Merge branch 'dev' into unify_name_label

2026-09-24 16:30:27 +08:00 · 2017-02-14 13:14:54 -05:00
parent c4491b2d83 9597b9b62b
commit 79c24066b4
44 changed files with 549 additions and 603 deletions
@@ -54,8 +54,10 @@ lib/galaxy/datatypes/dataproviders/__init__.py
 lib/galaxy/datatypes/data.py
 lib/galaxy/datatypes/display_applications/__init__.py
 lib/galaxy/datatypes/display_applications/util.py
+lib/galaxy/datatypes/genetics.py
 lib/galaxy/datatypes/images.py
 lib/galaxy/datatypes/__init__.py
+lib/galaxy/datatypes/interval.py
 lib/galaxy/datatypes/metadata.py
 lib/galaxy/datatypes/msa.py
 lib/galaxy/datatypes/ngsindex.py
@@ -36,6 +36,12 @@ docker-compose down | true
 docker-compose build galaxy
 docker-compose up -d

+function tear_down {
+    docker-compose down
+}
+
+trap tear_down EXIT
+
 for service_name in postgres galaxy selenium
 do
    echo "Waiting on service ${service_name}"
@@ -79,7 +85,6 @@ do
                docker logs "${container_id}"
                echo "---"
            done
-            docker-compose down | true
            exit 1
        fi
    done
@@ -105,6 +110,4 @@ exit_code=$?

 cd $TEST_DIRECTORY

-docker-compose down
-
 exit $exit_code
@@ -13,7 +13,9 @@ lib/galaxy/datatypes/constructive_solid_geometry.py
 lib/galaxy/datatypes/converters/
 lib/galaxy/datatypes/dataproviders/
 lib/galaxy/datatypes/data.py
+lib/galaxy/datatypes/genetics.py
 lib/galaxy/datatypes/images.py
+lib/galaxy/datatypes/interval.py
 lib/galaxy/datatypes/msa.py
 lib/galaxy/datatypes/ngsindex.py
 lib/galaxy/datatypes/proteomics.py
@@ -17,11 +17,10 @@ deeper discussion of some of these points - please see the
 If you have an idea for a feature to add or an approach for a bugfix,
 it is best to communicate with Galaxy developers early. The most
 common venues for this are
-[GitHub issues](https://github.com/galaxyproject/galaxy/issues) and the
-[Galaxy and Tool Shed Trello boards](https://wiki.galaxyproject.org/Issues).
+[GitHub issues](https://github.com/galaxyproject/galaxy/issues).
 Browse through existing GitHub issues and Trello cards and if one seems related,
-comment on it. We also maintain a [card](https://trello.com/c/eFdPIdIB) with
-links to smaller issues we believe would make the best entry points for new
+comment on it. We also maintain a [tag](https://github.com/galaxyproject/galaxy/issues?q=is%3Aissue+is%3Aopen+label%3Afriendliness%2Ffriendly) on Github for
+smaller issues we believe would make the best entry points for new
 developers.
 Galaxy developers are generally available via
 [IRC](https://wiki.galaxyproject.org/GetInvolved#IRC_Channel) and on
@@ -130,25 +130,25 @@ define( [ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view
            // button for version selection
            var versions_button = new Ui.ButtonMenu({
                icon    : 'fa-cubes',
-                title   : (!options.narrow && 'Versions') || null,
+                title   : ( !options.narrow && 'Versions' ) || null,
                tooltip : 'Select another tool version'
            });
-            if (!options.sustain_version && options.versions && options.versions.length > 1) {
-                for (var i in options.versions) {
-                    var version = options.versions[i];
-                    if (version != options.version) {
+            if ( !options.sustain_version && options.versions && options.versions.length > 1 ) {
+                for ( var i in options.versions ) {
+                    var version = options.versions[ i ];
+                    if ( version != options.version ) {
                        versions_button.addMenu({
                            title   : 'Switch to ' + version,
                            version : version,
                            icon    : 'fa-cube',
                            onclick : function() {
                                // here we update the tool version (some tools encode the version also in the id)
-                                var id = options.id.replace(options.version, this.version);
+                                var id = options.id.replace( options.version, this.version );
                                var version = this.version;
                                // queue model request
                                self.deferred.reset();
-                                self.deferred.execute(function(process) {
-                                    self._buildModel(process, {id: id, version: version})
+                                self.deferred.execute( function( process ) {
+                                    self._buildModel( process, { id : id, version : version } )
                                });
                            }
                        });
@@ -161,83 +161,64 @@ define( [ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view
            // button for options e.g. search, help
            var menu_button = new Ui.ButtonMenu({
                icon    : 'fa-caret-down',
-                title   : (!options.narrow && 'Options') || null,
+                title   : ( !options.narrow && 'Options' ) || null,
                tooltip : 'View available options'
            });
-            if(options.biostar_url) {
+            if ( options.biostar_url ) {
                menu_button.addMenu({
                    icon    : 'fa-question-circle',
                    title   : 'Question?',
-                    tooltip : 'Ask a question about this tool (Biostar)',
                    onclick : function() {
-                        window.open(options.biostar_url + '/p/new/post/');
+                        window.open( options.biostar_url + '/p/new/post/' );
                    }
                });
                menu_button.addMenu({
                    icon    : 'fa-search',
                    title   : 'Search',
-                    tooltip : 'Search help for this tool (Biostar)',
                    onclick : function() {
-                        window.open(options.biostar_url + '/local/search/page/?q=' + options.name);
+                        window.open( options.biostar_url + '/local/search/page/?q=' + options.name );
                    }
                });
            };
            menu_button.addMenu({
                icon    : 'fa-share',
                title   : 'Share',
-                tooltip : 'Share this tool',
                onclick : function() {
-                    prompt('Copy to clipboard: Ctrl+C, Enter', window.location.origin + Galaxy.root + 'root?tool_id=' + options.id);
+                    prompt( 'Copy to clipboard: Ctrl+C, Enter', window.location.origin + Galaxy.root + 'root?tool_id=' + options.id );
                }
            });

            // add admin operations
-            if (Galaxy.user && Galaxy.user.get('is_admin')) {
+            if ( Galaxy.user && Galaxy.user.get( 'is_admin' ) ) {
                menu_button.addMenu({
                    icon    : 'fa-download',
                    title   : 'Download',
-                    tooltip : 'Download this tool',
                    onclick : function() {
                        window.location.href = Galaxy.root + 'api/tools/' + options.id + '/download';
                    }
                });
-            }
-
-            // add admin operations for tool XML reloading
-            if (Galaxy.user && Galaxy.user.get('is_admin')) {
                menu_button.addMenu({
                    icon    : 'fa-refresh',
-                    title   : 'Reload Tool XML',
-                    tooltip : 'Reload tool XML file',
+                    title   : 'Reload XML',
                    onclick : function() {
-                        var modalMessage = new Ui.Modal.View();
-                        $.ajax({
-                            url: '/api/tools/' + options.id + '/reload',
-                            type: "GET",
-                        }).done(function(data){
-                            modalMessage.show({
-                                title   : data.done ? 'Tool XML Reload' : 'Tool XML Reload Error',
-                                body    : data.done ? data.done : data.error,
-                                buttons : { 'Close' : function() { modalMessage.hide() } }
-                            });
-                            window.setTimeout(function(){modalMessage.hide();}, 2000);
-                        }).fail(function(error){
-                            modalMessage.show({
-                                title: "Tool XML Reload AJAX Error",
-                                body: options.id + " " + error,
-                                buttons : { 'Close' : function() { modalMessage.hide() } }
-                            });
+                        Utils.get({
+                            url     : Galaxy.root + 'api/tools/' + options.id + '/reload',
+                            success : function( response ) {
+                                self.message.update( { persistent : false, message : 'Tool XML has been reloaded.', status : 'success' } );
+                            },
+                            error   : function( response ) {
+                                self.message.update( { persistent : false, message : response.err_msg, status : 'danger' } );
+                            }
                        });
                    }
                });
            }

            // button for version selection
-            if (options.requirements && options.requirements.length > 0) {
+            if ( options.requirements && options.requirements.length > 0 ) {
                menu_button.addMenu({
                    icon    : 'fa-info-circle',
                    title   : 'Requirements',
-                    tooltip : 'Display tool requirements',
                    onclick : function() {
                        if ( !this.requirements_visible || self.portlet.collapsed ) {
                            this.requirements_visible = true;
@@ -252,13 +233,12 @@ define( [ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view
            }

            // add toolshed url
-            if (options.sharable_url) {
+            if ( options.sharable_url ) {
                menu_button.addMenu({
                    icon    : 'fa-external-link',
                    title   : 'See in Tool Shed',
-                    tooltip : 'Access the repository',
                    onclick : function() {
-                        window.open(options.sharable_url);
+                        window.open( options.sharable_url );
                    }
                });
            }
@@ -61,6 +61,7 @@
    <datatype extension="bowtie_base_index" type="galaxy.datatypes.ngsindex:BowtieBaseIndex" mimetype="text/html" display_in_upload="false"/>
    <datatype extension="csfasta" type="galaxy.datatypes.sequence:csFasta" display_in_upload="true"/>
    <datatype extension="data" type="galaxy.datatypes.data:Data" mimetype="application/octet-stream" max_optional_metadata_filesize="1048576" />
+    <datatype extension="d3_hierarchy" type="galaxy.datatypes.text:Json" mimetype="application/json" subclass="true" display_in_upload="false"/>
    <datatype extension="data_manager_json" type="galaxy.datatypes.text:Json" mimetype="application/json" subclass="true" display_in_upload="false"/>
    <datatype extension="dbn" type="galaxy.datatypes.sequence:DotBracket" display_in_upload="true" description="Dot-Bracket format is a text-based format for storing both an RNA sequence and its corresponding 2D structure." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Dbn"/>
    <datatype extension="fai" type="galaxy.datatypes.tabular:Tabular" display_in_upload="true" subclass="true" description="A Fasta Index File is a text file consisting of lines each with five TAB-delimited columns : Name, Length, offset, linebases, Linewidth" description_url="http://www.htslib.org/doc/faidx.html" />
@@ -447,7 +448,7 @@
    <datatype extension="embl" type="galaxy.datatypes.data:Text" subclass="true"/>
    <datatype extension="fitch" type="galaxy.datatypes.data:Text" subclass="true"/>
    <datatype extension="gcg" type="galaxy.datatypes.data:Text" subclass="true"/>
-    <datatype extension="genbank" type="galaxy.datatypes.sequence:Genbank" edam_format="format_1936" display_in_upload="True"/>
+    <datatype extension="genbank" type="galaxy.datatypes.sequence:Genbank" display_in_upload="True"/>
    <datatype extension="hennig86" type="galaxy.datatypes.data:Text" subclass="true"/>
    <datatype extension="ig" type="galaxy.datatypes.data:Text" subclass="true"/>
    <datatype extension="jackknifer" type="galaxy.datatypes.data:Text" subclass="true"/>
@@ -145,8 +145,6 @@ steps:
      element: "#current-history-panel .fa-times:eq(0)"
      intro: "Delete your dataset by clicking the x-button."
      position: "left"
-      #postclick:
-      #  - "#current-history-panel .fa-times:eq(0)"

    - title: "Dataset information"
      element: "div.title-bar.clear:eq(0)"
@@ -156,11 +154,9 @@ steps:
        - "div.title-bar.clear:eq(0)"

    - title: "Re-run tool"
-      element: ".fa-refresh:eq(1)"
+      element: "#current-history-panel .fa-refresh:first"
      intro: "By clicking the reload button, you can re-run your tool again (e.g. with different parameters or on another dataset)."
      position: "left"
-      preclick:
-        - ".fa-refresh:eq(1)"


    - title: "Panel collapse"
@@ -107,13 +107,6 @@ steps:
      postclick:
        - "#current-history-panel > ul.list-items > div:nth-child(1) > div.warnings > div > a"

-    - element: "#current-history-panel > div.controls > .subtitle .toggle-deleted-link"
-      title: "Hiding all deleted datasets"
-      intro: "Hiding datasets that were previously deleted works in the same way."
-      position: "bottom"
-      preclick:
-        - "#current-history-panel > div.controls > .subtitle .toggle-deleted-link"
-
    - element: "#current-history-panel > div.controls > div.title > div"
      title: "Change your History name"
      intro: "You can change the history name clicking on the title."
@@ -1,13 +1,20 @@
 #!/bin/sh

 if [ -z "$3" ]; then
-    echo "usage: check_galaxy <server> <username> <password>"
+    echo "usage: check_galaxy <server> <username> <password> <handler>"
    exit 3
 fi

-here=`dirname $0`
-var="$HOME/.check_galaxy/$1"
+# not sure why this is necessary, but nagios' $HOME is /root, despite not running as root.
+HOME=/var/lib/nagios
+export HOME

+here=`dirname $0`
+host=`basename $1`
+handler="$4"
+var="$HOME/.check_galaxy/$host/$handler"
+
+mkdir -p $var
 touch $var/iterations
 iterations=`cat $var/iterations`
 if [ -z "$iterations" ]; then
@@ -15,7 +22,7 @@ if [ -z "$iterations" ]; then
 fi

 new_history=''
-if [ $iterations -gt 96 ]; then
+if [ $iterations -gt 64 ]; then
    new_history='-n'
    echo 0 > $var/iterations
 else
@@ -23,17 +30,17 @@ else
 fi

 date >> $var/log
-status=`python $here/check_galaxy.py $new_history $1 $2 $3 2>&1 | tee -a $var/log | tail -n 1`
+status=`python $here/check_galaxy.py $new_history $1 $2 $3 $4 2>&1 | tee -a $var/log | tail -n 1`

 echo "$status"

 case "$status" in
-    "Exception: Tool never finished")
-        exit 1
-        ;;
-    "OK")
+    "OK: "*)
        exit 0
        ;;
+    "AssertionError: "*)
+        exit 1
+        ;;
    *)
        exit 2
        ;;
@@ -5,7 +5,6 @@ via the check_galaxy.sh script in Galaxy's cron/ directory.
 """
 from __future__ import print_function

-import filecmp
 import formatter
 import getopt
 import htmllib
@@ -13,13 +12,13 @@ import json
 import os
 import socket
 import sys
-import tempfile
 import time
+import urllib2
 import warnings
+from user import home

 with warnings.catch_warnings():
    warnings.simplefilter('ignore')
-    import twill
    import twill.commands as tc

 # options
@@ -28,38 +27,10 @@ if "DEBUG" in os.environ:
 else:
    debug = False

-test_data_dir = os.path.join( os.path.dirname( __file__ ), 'check_galaxy_data' )
-# what tools to run - not so pretty
-tools = {
-    "Extract+genomic+DNA+1":
-    [
-        {
-            "inputs":
-            (
-                {
-                    "file_path": os.path.join( test_data_dir, "1.bed" ),
-                    "dbkey": "hg17",
-                },
-
-            )
-        },
-        { "check_file": os.path.join( test_data_dir, "extract_genomic_dna_out1.fasta" ) },
-        {
-            "tool_run_options":
-            {
-                "input": "1.bed",
-                "interpret_features": "yes",
-                "index_source": "cached",
-                "out_format": "fasta"
-            }
-        }
-    ]
-}
-

 # handle arg(s)
 def usage():
-    sys.exit("usage: check_galaxy.py <server> <username> <password>")
+    sys.exit("usage: check_galaxy.py <server> <username> <password> <handler>")


 try:
@@ -72,6 +43,8 @@ if len( args ) < 1:
 server = args[0]
 username = args[1]
 password = args[2]
+handler = args[3]
+warntime = 240

 new_history = False
 for o, a in opts:
@@ -83,69 +56,70 @@ for o, a in opts:
        usage()

 # state information
-var_dir = os.path.join( os.path.expanduser('~'), ".check_galaxy", server )
+var_dir = os.path.join( home, ".check_galaxy", server.replace('http://', '').replace('https://', ''), handler )
 if not os.access( var_dir, os.F_OK ):
    os.makedirs( var_dir, 0o700 )

 # default timeout for twill browser is never
-socket.setdefaulttimeout(300)
+socket.setdefaulttimeout(60)

 # user-agent
-tc.agent("Mozilla/5.0 (compatible; check_galaxy/0.1)")
+tc.agent("Mozilla/5.0 (compatible; check_galaxy/0.2)")
 tc.config('use_tidy', 0)


 class Browser:
    def __init__(self):
        self.server = server
-        self.tool = None
-        self.tool_opts = None
+        self.handler = handler
+        self.waited = -1
+        self.tool = 'echo_' + handler
        self._hda_id = None
        self._hda_state = None
        self._history_id = None
-        self.check_file = None
+        if not self.server.startswith('http'):
+            self.server = 'http://' + self.server
        self.cookie_jar = os.path.join( var_dir, "cookie_jar" )
        dprint("cookie jar path: %s" % self.cookie_jar)
        if not os.access(self.cookie_jar, os.R_OK):
            dprint("no cookie jar at above path, creating")
            tc.save_cookies(self.cookie_jar)
        tc.load_cookies(self.cookie_jar)
+        self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(tc.get_browser().cj))

    def get(self, path):
-        tc.go("http://%s%s" % (self.server, path))
+        tc.go("%s%s" % (self.server, path))
        tc.code(200)

+    def req(self, path, data=None, method=None):
+        url = self.server + path
+        if data:
+            req = urllib2.Request(url, headers={'Content-Type': 'application/json'}, data=json.dumps(data))
+        else:
+            req = urllib2.Request(url, headers={'Content-Type': 'application/json'})
+        if method:
+            req.get_method = lambda: method
+        res = self.opener.open( req )
+        print('==> at %s (%s)' % (url, method or 'GET'))
+        assert res.getcode() == 200, url
+        return res
+
    def reset(self):
-        self.tool = None
-        self.tool_opts = None
        self._hda_id = None
        self._hda_state = None
        self._history_id = None
-        self.check_file = None
        if new_history:
-            self.get("/history/delete_current")
-            tc.save_cookies(self.cookie_jar)
+            self.delete_history()
+            self.create_history()
        self.delete_datasets()

-    def check_redir(self, url):
-        try:
-            tc.get_browser()._browser.set_handle_redirect(False)
-            tc.go(url)
-            tc.code(302)
-            tc.get_browser()._browser.set_handle_redirect(True)
-            dprint( "%s is returning redirect (302)" % url )
-            return(True)
-        except twill.errors.TwillAssertionError as e:
-            tc.get_browser()._browser.set_handle_redirect(True)
-            dprint( "%s is not returning redirect (302): %s" % (url, e) )
-            code = tc.browser.get_code()
-            if code == 502:
-                sys.exit("Galaxy is down (code 502)")
-            return False
+    def delete_history(self):
+        # note, this could cause a history to be created and then deleted.  i don't care.
+        self.req( '/api/histories/%s' % self.history_id, method='DELETE' )

    def login(self, user, pw):
        self.get("/user/login")
-        tc.fv("1", "email", user)
+        tc.fv("1", "login", user)
        tc.fv("1", "password", pw)
        tc.submit("Login")
        tc.code(200)
@@ -153,50 +127,37 @@ class Browser:
            # uh ohs, fail
            p = userParser()
            p.feed(tc.browser.get_html())
-            if p.no_user:
-                dprint("user does not exist, will try creating")
-                self.create_user(user, pw)
-            elif p.bad_pw:
+            if p.bad_pw:
                raise Exception("Password is incorrect")
            else:
                raise Exception("Unknown error logging in")
        tc.save_cookies(self.cookie_jar)

-    def create_user(self, user, pw):
-        self.get("/user/create")
-        tc.fv("1", "email", user)
-        tc.fv("1", "password", pw)
-        tc.fv("1", "confirm", pw)
-        tc.submit("Submit")
-        tc.code(200)
-        if len(tc.get_browser().get_all_forms()) > 0:
-            p = userParser()
-            p.feed(tc.browser.get_html())
-            if p.already_exists:
-                raise Exception('The user you were trying to create already exists')
-
-    def upload(self, input):
-        self.get("/tool_runner/index?tool_id=upload1")
-        tc.fv("1", "file_type", "bed")
-        tc.fv("1", "dbkey", input.get('dbkey', '?'))
-        tc.formfile("1", "file_data", input['file_path'])
-        tc.submit("runtool_btn")
-        tc.code(200)
-
    def runtool(self):
-        self.get("/tool_runner/index?tool_id=%s" % self.tool)
-        for k, v in self.tool_opts.items():
-            tc.fv("1", k, v)
-        tc.submit("runtool_btn")
-        tc.code(200)
+        path = '/api/tools'
+        data = { 'tool_id' : self.tool,
+                 'history_id' : self.history_id,
+                 'inputs' : { 'echo' : self.handler } }
+        res = self.req(path, data=data)
+        dprint(json.loads(res.read()))

    @property
    def history_id(self):
        if self._history_id is None:
            self.get('/api/histories')
-            self._history_id = json.loads(tc.browser.get_html())[0]['id']
+            for history in json.loads(tc.browser.get_html()):
+                # find an undeleted history named the same as the handler
+                if history['name'] == self.handler:
+                    self._history_id = history['id']
+                    break
+            else:
+                self.create_history()
        return self._history_id

+    def create_history(self):
+        res = self.req('/api/histories', data={'name' : handler})
+        self._history_id = json.loads(res.read())['id']
+
    @property
    def history_contents(self):
        self.get('/api/histories/%s/contents' % self.history_id)
@@ -244,7 +205,8 @@ class Browser:
    def wait(self):
        sleep_amount = 1
        count = 0
-        maxiter = 16
+        maxiter = 20
+        start = time.time()
        while count < maxiter:
            count += 1
            if not self.history_state_terminal:
@@ -252,8 +214,8 @@ class Browser:
                sleep_amount += 1
            else:
                break
-        if count == maxiter:
-            raise Exception("Tool never finished")
+        self.waited = time.time() - start
+        assert count < maxiter, "Job timeout, waited %.2f seconds" % self.waited

    def check_state(self):
        if self.hda_state != "ok":
@@ -261,36 +223,31 @@ class Browser:
            print(tc.browser.get_html())
            raise Exception("HDA %s NOT OK: %s" % (self.hda_id, self.hda_state))

-    def diff(self):
-        self.get("/datasets/%s/display?to_ext=%s" % (self.hda_id, self.tool_opts.get('out_format', 'fasta')))
-        data = tc.browser.get_html()
-        tmp = tempfile.mkstemp()
-        dprint("tmp file: %s" % tmp[1])
-        tmpfh = os.fdopen(tmp[0], 'w')
-        tmpfh.write(data)
-        tmpfh.close()
-        if filecmp.cmp(tmp[1], self.check_file):
-            dprint("Tool output is as expected")
+    def check_hda_content(self):
+        self.get("/datasets/%s/display?to_ext=txt" % self.hda_id)
+        data = tc.browser.get_html().strip()
+        if data == self.handler:
+            dprint("Tool output is correct: %s" % data)
        else:
-            if not debug:
-                os.remove(tmp[1])
+            dprint("EXPECTED: %s" % self.handler)
+            dprint("GOT: %s" % data)
            raise Exception("Tool output differs from expected")
-        if not debug:
-            os.remove(tmp[1])

    def delete_datasets(self):
        for hda in self.undeleted_hdas:
-            self.get('/datasets/%s/delete' % hda['id'])
+            path = '/api/histories/%s/contents/%s' % (self.history_id, hda['id'])
+            self.req(path, method='DELETE')
        hdas = [hda['id'] for hda in self.undeleted_hdas]
        if hdas:
            print("Remaining datasets ids:", " ".join(hdas))
            raise Exception("History still contains datasets after attempting to delete them")

-    def check_if_logged_in(self):
-        self.get("/user?cntrller=user")
-        p = loggedinParser()
-        p.feed(tc.browser.get_html())
-        return p.logged_in
+    def check_if_logged_in(self, user):
+        try:
+            return json.loads(self.req('/api/users').read())[0]['email'] == user
+        except Exception as e:
+            print('Exception checking if logged in: %s' % str(e))
+            return False


 class userParser(htmllib.HTMLParser):
@@ -324,26 +281,6 @@ class userParser(htmllib.HTMLParser):
                self.already_exists = True


-class loggedinParser(htmllib.HTMLParser):
-    def __init__(self):
-        htmllib.HTMLParser.__init__(self, formatter.NullFormatter())
-        self.in_p = False
-        self.logged_in = False
-
-    def start_p(self, attrs):
-        self.in_p = True
-
-    def end_p(self):
-        self.in_p = False
-
-    def handle_data(self, data):
-        if self.in_p:
-            if data == "You are currently not logged in.":
-                self.logged_in = False
-            elif data.startswith( "You are currently logged in as " ):
-                self.logged_in = True
-
-
 def dprint(str):
    if debug:
        print(str)
@@ -356,39 +293,22 @@ if __name__ == "__main__":
    b = Browser()

    # login (or not)
-    if b.check_if_logged_in():
+    if b.check_if_logged_in(username):
        dprint("we are already logged in (via cookies), hooray!")
    else:
        dprint("not logged in... logging in")
        b.login(username, password)

-    for tool, params in tools.items():
+    # make sure history and state is clean
+    b.reset()

-        check_file = ""
+    b.runtool()
+    b.wait()
+    b.check_state()
+    b.check_hda_content()
+    b.delete_datasets()

-        # make sure history and state is clean
-        b.reset()
-        b.tool = tool
+    assert b.waited <= warntime, "Warning: Job runtime: %.2f" % b.waited

-        # get all the tool run conditions
-        for dict in params:
-            for k, v in dict.items():
-                if k == 'inputs':
-                    for input in v:
-                        b.upload(input)
-                    b.wait()
-                elif k == 'check_file':
-                    b.check_file = v
-                elif k == 'tool_run_options':
-                    b.tool_opts = v
-                else:
-                    raise Exception("Unknown key in tools dict: %s" % k)
-
-        b.runtool()
-        b.wait()
-        b.check_state()
-        b.diff()
-        b.delete_datasets()
-
-    print("OK")
+    print("OK: Job runtime: %.2f" % b.waited)
    sys.exit(0)
@@ -1,18 +1,21 @@
 layout: true
-class: inverse, middle, large
+name: left-aligned
+class: left, middle

 ---
-class: special
+
+layout: true
+class: center, middle
+
+---
+
+layout: introduction_slides
+topic_name: Galaxy Architecture
+
 # Galaxy Architecture

-Nate, James, John, Rémi
-
-.footnote[\#usegalaxy / @galaxyproject]
-
 ---

-class: larger
-
 ### Please Interrupt!

 We're here to answer your questions about Galaxy architecture!
@@ -23,13 +26,13 @@ We're here to answer your questions about Galaxy architecture!

 ---

-class: larger
+**Gitter:** [galaxyproject/Lobby](https://gitter.im/galaxyproject/Lobby)

-**IRC:** irc.freenode.net#galaxyproject
+**IRC:** [irc.freenode.net#galaxyproject](https://webchat.freenode.net/?channels=galaxyproject)

-**GitHub:** github.com/galaxyproject
+**GitHub:** [github.com/galaxyproject](https://github.com/galaxyproject)

-**Twitter:**: #usegalaxy, @galaxyproject
+**Twitter:** #usegalaxy, @galaxyproject

 ---

@@ -45,33 +48,37 @@ Contribution guidelines: http://bit.ly/gx-CONTRIBUTING-md

 ---

-github.com/galaxyproject/**galaxy**
+[github.com/galaxyproject/**galaxy**](https://github.com/galaxyproject/galaxy)

-The main Galaxy application. Web interface, database model, job running, etc. Also includes other web applications including the **ToolShed** and **Reports**
+The main Galaxy application.
+
+Web interface, database model, job running, etc...
+
+Also includes other web applications including the **ToolShed** and **Reports**

 ---

-github.com/galaxyproject/**cloudman**
+[github.com/galaxyproject/**cloudman**](https://github.com/galaxyproject/cloudman)

 Galaxy CloudMan - a web application which manages a Galaxy cluster in
 the cloud.

-github.com/galaxyproject/**cloudlaunch**
+[github.com/galaxyproject/**cloudlaunch**](https://github.com/galaxyproject/cloudlaunch)

 CloudLaunch web application to make it wasy to launch images on a cloud, drives *https://launch.usegalaxy.org*

 ---

-github.com/galaxyproject/**tools-iuc**
+[github.com/galaxyproject/**tools-iuc**](https://github.com/galaxyproject/tools-iuc)

-Galaxy tools maintained by *iuc* (the "Intergalactic Utilities Commission").
+Galaxy tools maintained by the *IUC* ("Intergalactic Utilities Commission").

 A variety of tools, generally of high quality including many of the core tools for Galaxy main.

 Demonstrates *current tool development best practices* - development on
 github and then deployed to test/main ToolSheds

-github.com/galaxyproject/**tools-devteam**
+[github.com/galaxyproject/**tools-devteam**](https://github.com/galaxyproject/tools-devteam)

 Many older tools appearing on usegalaxy.org.

@@ -99,7 +106,7 @@ Other repositories with high quality tools:

 ---

-github.com/galaxyproject/**starforge**
+[github.com/galaxyproject/**starforge**](https://github.com/galaxyproject/starforge)

 Build Galaxy Tool dependencies for the ToolShed in Docker containers

@@ -107,11 +114,12 @@ Build Galaxy framework dependencies as Python wheels

 ---

-github.com/galaxyproject/**planemo**
+[github.com/galaxyproject/**planemo**](https://github.com/galaxyproject/planemo)

 Commande line utilities to assist in the development of Galaxy tools.
-Linting, testing, deploying to ToolSheds... *The best practice approach
-for Galaxy tool development!*
+Linting, testing, deploying to ToolSheds...
+
+*The best practice approach for Galaxy tool development!*

 github.com/galaxyproject/**planemo-machine**

@@ -122,15 +130,15 @@ container, virtual machines, Google compute images

 github.com/galaxyproject/**{ansible-\*, \*-playbook}**

-Ansible components to automate almost every aspect of Galaxy installation and maintenance.
+[Ansible](https://www.ansible.com/) components to automate almost every aspect of Galaxy installation and maintenance.

 Ansible is an advanced configuration management system

-These playbooks are used to maintain Galaxy main, cloud images, virtual machines, ...
+These playbooks are used to maintain Galaxy main, cloud and Docker images, virtual machines, ...

 ---

-github.com/galaxyproject/**pulsar**
+[github.com/galaxyproject/**pulsar**](https://github.com/galaxyproject/pulsar)

 Distributed job execution engine for Galaxy.

@@ -142,7 +150,7 @@ Can act as its own queuing system or access an existing cluster DRM.

 ---

-github.com/galaxyproject/**bioblend**
+[github.com/galaxyproject/**bioblend**](https://github.com/galaxyproject/bioblend)

 Official Python client for the Galaxy, ToolShed, and CloudMan APIs.

@@ -150,15 +158,15 @@ Best documented path to scripting the Galaxy API.

 ---

- github.com/galaxyproject/**blend4php**
- github.com/**jmchilton/blend4j**
- github.com/**chapmanb/clj-blend**
+- [github.com/galaxyproject/**blend4php**](https://github.com/galaxyproject/blend4php)
+- [github.com/**jmchilton/blend4j**](https://github.com/jmchilton/blend4j)
+- [github.com/**chapmanb/clj-blend**](https://github.com/chapmanb/clj-blend)

 Galaxy API bindings for other languages.

 ---

-github.com/**bgruening/docker-galaxy-stable**
+[github.com/**bgruening/docker-galaxy-stable**](https://github.com/bgruening/docker-galaxy-stable)

 High quality Docker containers for stable Galaxy environments.

@@ -235,7 +243,7 @@ the `run.sh` should "just work" and should work quickly.

 So by default Galaxy does not require:

- - Compilation - it fetches *binary wheels*.
+ - Compilation - it fetches *binary wheels* for your platform.
 - A job manager - Galaxy can act as one.
 - An external database server - Galaxy can use an sqlite database.
 - A web proxy or external Python web server.
@@ -251,7 +259,7 @@ So by default Galaxy does not require:
 ???

 Workflow, Data Libraries, Visualization, History, Tool Menu,
-Many Grids
+Many Grids, User and preference management.

 ---

@@ -267,8 +275,7 @@ class: white

 ???

-User management and admin things, Reports and Tool Shed
-Webapp
+Admin things, Reports and Tool Shed Webapp

 ---

@@ -291,6 +298,8 @@ Webapp

 ---

+template: left-aligned
+
 ### Galaxy WSGI Middleware

 A WSGI function:
@@ -303,7 +312,7 @@ A WSGI function:

 ---

-class: normal
+template: left-aligned

 ### Galaxy's WSGI Middleware

@@ -320,11 +329,11 @@ Middleware configured in `galaxy.webapps.galaxy.buildapp#wrap_in_middleware`.

 ---

-background-image: url(images/webapp.plantuml.svg)
+![webapp](images/webapp.plantuml.svg)

 ---

-class: normal
+template: left-aligned

 ### Routes

@@ -348,7 +357,7 @@ Uses popular Routes library (https://pypi.python.org/pypi/Routes).

 ---

-class: normal
+template: left-aligned

 Simplified `handle_request` from `lib/galaxy/web/framework/base.py`.

@@ -413,8 +422,6 @@ def handle_request(self, environ, start_response):

 ---

-class: white, widen_image
-
 ![SQLAlchemy Architecture](images/sqla_arch_small.png)

 ---
@@ -432,7 +439,6 @@ class: white, widen_image

 ---

-class: white, narrow_image
 ![Galaxy Schema](images/galaxy_schema.png)

 ### Database Diagram
@@ -494,8 +500,6 @@ Slides for datatypes, example of meta data definitions...

 ---

-class: normal
-
 ### Object Store

 .strike[```
@@ -514,10 +518,12 @@ class: normal

 ---

-background-image: url(images/objectstore.plantuml.svg)
+![Object Store](images/objectstore.plantuml.svg)

 ---

+template: left-aligned
+
 ### Visualization Plugins

 Adding new visualizations to a Galaxy instance
@@ -528,7 +534,7 @@ Adding new visualizations to a Galaxy instance

 ---

-class: smaller
+class: reduce70

 ```xml
 <?xml version="1.0" encoding="UTF-8"?>
@@ -551,6 +557,8 @@ class: smaller

 ---

+template: left-aligned
+
 ### Visualization Examples

 All in `config/plugins/visualizations`:
@@ -586,6 +594,8 @@ Build a UI that accesses that process through a proxy

 ---

+template: left-aligned
+
 ### Interactive Environments - Examples

 All in `config/plugins/interactive_environments`:
@@ -612,6 +622,8 @@ in a manager instead of in the model.

 ---

+template: left-aligned
+
 ### Client Directories

 - Source stylesheets and JavaScript in `client/galaxy/{style|scripts}`
@@ -642,11 +654,13 @@ npm-deps: ## Install NodeJS dependencies.

 ---

+template: left-aligned
+
 ### grunt

 Build tool for node/JavaScript, tasks in `client/Gruntfile.js`. Default task is

-.smaller[```grunt.registerTask( 'default', [ 'check-modules', 'uglify', 'webpack' ] );```]
+.center[`grunt.registerTask( 'default', [ 'check-modules', 'uglify', 'webpack' ] );`]

 - `check-modules` Verifies node dependencies are correct and exact.
 - [`uglify`](https://github.com/mishoo/UglifyJS) Compresses JavaScript modules in `client` and move to `static` and creates source maps.
@@ -656,6 +670,8 @@ Build tool for node/JavaScript, tasks in `client/Gruntfile.js`. Default task is

 ---

+template: left-aligned
+
 ### JavaScript Modules - The Problem

 From http://requirejs.org/docs/why.html:
@@ -668,6 +684,8 @@ From http://requirejs.org/docs/why.html:

 ---

+template: left-aligned
+
 ### JavaScript Modules - The Solution

 From http://requirejs.org/docs/why.html:
@@ -681,8 +699,6 @@ RequireJS an implementation of AMD.

 ---

-class: normal
-
 ### JavaScript Modules - Galaxy AMD Example

 ```javascript
@@ -737,6 +753,8 @@ class: white

 ---

+template: left-aligned
+
 ### Dependencies - Python

 `script/common_startup.sh` sets up a `virtualenv` with required dependencies in `$GALAXY_ROOT/.venv` (or `$GALAXY_VIRTUAL_ENV` if set).
@@ -750,6 +768,8 @@ class: white

 ---

+template: left-aligned
+
 ### Dependencies - JavaScript

 These come bundled with Galaxy, so do not need to be fetched at runtime.
@@ -764,8 +784,6 @@ These come bundled with Galaxy, so do not need to be fetched at runtime.

 ---

-class: normal
-
 ### Cloning Galaxy

 ```
@@ -785,7 +803,7 @@ $ sh run.sh

 ---

-class: normal
+class: reduce90

 ### Copying Configs

@@ -823,7 +841,7 @@ Successfully installed pip-8.1.2

 ---

-class: normal
+class: reduce70

 ### Installing Dependencies

@@ -851,7 +869,7 @@ Successfully installed Babel-2.0 Beaker-1.7.0 Cheetah-2.4.4 Fabric-1.10.2 Mako-1

 ---

-class: smaller
+class: reduce70

 ### Initial Debugging as App Starts

@@ -871,7 +889,7 @@ galaxy.app DEBUG 2016-06-23 19:11:51,956 Using "galaxy.ini" config file:

 ---

-class: normal
+class: reduce70

 ### Database Migrations

@@ -905,7 +923,7 @@ Everything after here happens every time

 ---

-class: smaller
+class: reduce70

 .code[```
 migrate.versioning.repository DEBUG 2016-06-23 19:13:35,635 Loading repository lib/tool_shed/galaxy_install/migrate...
@@ -928,7 +946,7 @@ galaxy.config INFO 2016-06-23 19:13:35,679 Install database targetting Galaxy's

 ---

-class: smaller
+class: reduce70

 .code[```
 galaxy.datatypes.registry DEBUG 2016-06-23 19:13:35,748 Loading datatypes from ./config/datatypes_conf.xml.sample
@@ -956,7 +974,7 @@ galaxy.datatypes.registry DEBUG 2016-06-23 19:13:35,824 Retrieved datatype modul

 ---

-class: smaller
+class: reduce70

 .code[```
 galaxy.datatypes.registry DEBUG 2016-06-23 19:13:35,824 Loaded sniffer for datatype 'galaxy.datatypes.mothur:Sabund'
@@ -1001,7 +1019,7 @@ galaxy.datatypes.registry DEBUG 2016-06-23 19:13:35,833 Loaded sniffer for datat

 ---

-class: smaller
+class: reduce70

 .code[```
 galaxy.datatypes.registry DEBUG 2016-06-23 19:13:35,833 Loaded build site 'ucsc': tool-data/shared/ucsc/ucsc_build_sites.txt with display sites: main,test,archaea,ucla
@@ -1014,7 +1032,7 @@ galaxy.datatypes.registry DEBUG 2016-06-23 19:13:35,834 Loaded build site 'rview

 ---

-class: smaller
+class: reduce70

 .code[```
 galaxy.tools.data INFO 2016-06-23 19:13:35,871 Could not find tool data tool-data/all_fasta.loc, reading sample
@@ -1031,7 +1049,7 @@ galaxy.tools.data DEBUG 2016-06-23 19:13:36,211 Loaded tool data table 'biom_sim

 ---

-class: normal
+class: reduce70

 ### Job Configuration, Citation Cache

@@ -1043,7 +1061,7 @@ beaker.container DEBUG 2016-06-23 19:13:36,278 data file ./database/citations/da

 ---

-class: smaller
+class: reduce70

 ### Load Toolbox

@@ -1062,7 +1080,7 @@ galaxy.tools.toolbox.base INFO 2016-06-23 19:13:36,497 Parsing the tool configur

 ---

-class: smaller
+class: reduce90

 ### Tool Dependency Resolution and Indexing 

@@ -1076,7 +1094,7 @@ galaxy.tools.search DEBUG 2016-06-23 19:13:37,789 Toolbox index finished. It too

 ---

-class: smaller
+class: reduce70

 ### Display Applications

@@ -1098,7 +1116,7 @@ galaxy.datatypes.registry DEBUG 2016-06-23 19:13:38,007 Adding inherited display

 ---

-class: smaller
+class: reduce70

 ### Datatype Converters

@@ -1117,7 +1135,7 @@ galaxy.datatypes.registry DEBUG 2016-06-23 19:13:38,099 Loaded converter: CONVER

 ---

-class: normal
+class: reduce90

 ### Special Tools

@@ -1129,7 +1147,7 @@ galaxy.tools.special_tools DEBUG 2016-06-23 19:13:38,108 Loaded history export t

 ---

-class: normal
+class: reduce70

 ### Vizualization Plugins

@@ -1148,8 +1166,6 @@ galaxy.web.base.pluginframework INFO 2016-06-23 19:13:38,114 VisualizationsRegis

 ---

-class: normal
-
 ### Tours

 .code[```
@@ -1160,7 +1176,7 @@ galaxy.tours INFO 2016-06-23 19:13:38,183 Loaded tour 'core.history'

 ---

-class: normal
+class: reduce90

 ### Job Handler and Runners

@@ -1178,7 +1194,7 @@ galaxy.jobs.handler INFO 2016-06-23 19:13:38,222 job handler queue started

 ---

-class: normal
+class: reduce70

 ### Ignore this...

@@ -1189,7 +1205,7 @@ galaxy.sample_tracking.external_service_types DEBUG 2016-06-23 19:13:38,230 Load

 ---

-class: normal
+class: reduce90

 ### Workflow Scheduler

@@ -1199,7 +1215,7 @@ galaxy.workflow.scheduling_manager DEBUG 2016-06-23 19:13:38,254 Starting workfl

 ---

-class: normal
+class: reduce90

 ### Controllers

@@ -1218,7 +1234,7 @@ galaxy.web.framework.base DEBUG 2016-06-23 19:13:38,728 Enabling 'workflow_tags'

 ---

-class: normal
+class: reduce90

 ### Middleware

@@ -1233,7 +1249,7 @@ galaxy.webapps.galaxy.buildapp DEBUG 2016-06-23 19:13:39,044 Enabling 'Request I

 ---

-class: normal
+class: reduce70

 ### Static Paths for Viz

@@ -1246,6 +1262,8 @@ galaxy.webapps.galaxy.buildapp DEBUG 2016-06-23 19:13:39,049 added url, path to

 ---

+class: reduce90
+
 ### It is Up!

 .code[```
@@ -2,14 +2,14 @@ skinparam handwritten true
 ' skinparam roundcorner 20

 skinparam class {
-    ArrowFontColor #FFEFD5
+    ArrowFontColor DarkOrange
    BackgroundColor #FFEFD5
    ArrowColor Orange
    BorderColor DarkOrange
 }

 skinparam object {
-    ArrowFontColor #FFEFD5
+    ArrowFontColor DarkOrange
    BackgroundColor #FFEFD5
    ArrowColor Orange
    BorderColor DarkOrange
@@ -22,7 +22,7 @@ skinparam note {

 skinparam sequence {
    ArrowColor Orange
-    ArrowFontColor #FFEFD5
+    ArrowFontColor DarkOrange
    ActorBorderColor DarkOrange
    ActorBackgroundColor #FFEFD5

@@ -1,99 +1,165 @@
-@import url(https://fonts.googleapis.com/css?family=Oxygen);
-@import url(https://fonts.googleapis.com/css?family=Consolas);
-@import url(https://fonts.googleapis.com/css?family=Droid+Serif);
@import url(https://fonts.googleapis.com/css?family=Yanone+Kaffeesatz);
+@import url(https://fonts.googleapis.com/css?family=Droid+Serif:400,700,400italic);
@import url(https://fonts.googleapis.com/css?family=Ubuntu+Mono:400,700,400italic);

-body {
-    font-family: 'Oxygen', 'PT Sans', Serif;
-}
+body { font-family: 'Droid Serif'; }
+
 h1, h2, h3 {
-    font-weight: bold;
-}
-h1 {
-    font-size: 3em;
-    color: #ff9300;
-}
-h2 { font-size: 2em; }
-
-.title h1, h2 {
-    margin-bottom: 0px;
-    margin-top: 0px;
+    font-family: 'Yanone Kaffeesatz';
+    font-weight: normal;
    text-align: center;
 }

-.title p {
-    margin-top: 50px;
-    text-align: center;
-}
-
-.centered {
-    text-align: center;
-}
-
-img {
-    max-width: 100%;
-}
-
-.widen_image img {
-    max-width: 100%;
+h3{
+    position: absolute;
+    top: 30px;
+    left: 0px;
    width: 100%;
 }

-.narrow_image img {
-    height: 500px;
+.packed h3{
+    position: absolute;
+    top: 10px;
+    left: 0px;
+    width: 100%;
+    margin-top:0px;
 }

-h3 {
-    font-size: 1.8em;
-    position: absolute;
-    top: .5em;
+.hljs-monokai .hljs {
+    display: block;
+    overflow-x: auto;
+    padding: .5em;
+    background: #272822;
+    color: #ddd;
+}
+
+.remark-code, .remark-inline-code {
+    font-family: 'Ubuntu Mono';
+    text-align: left;
+}
+
+.remark-code{
+    font-size: 18px;
+}
+
+.remark-code-line {
+    min-height: 1em;
+}
+
+ul, ol {
+    text-align: left;
+}
+
+img{
+    max-height: 400px;
+    max-width: 700px;
+}
+
+.image-10 img {
+  width: 10%;
+}
+
+.image-25 img {
+  width: 25%;
+}
+
+.image-50 img {
+  width: 50%;
+}
+
+.image-75 img {
+  width: 75%;
 }

 .footnote {
    position: absolute;
-    bottom: 3em;
+    bottom: 60px;
+    left: 0px;
+    width: 100%;
+    font-size: 15px;
+    color: #444444;
 }

-.large {
-    font-size: 1.6em;
+.my-footer {
+    position: absolute;
+    bottom: 0px;
+    left: 0px;
+    height: 50px;
+    width: 100%;
 }

-.larger {
-    font-size: 2em;
+.my-footer span {
+    font-size: 10pt;
+    position: absolute;
+    left: 15px;
+    bottom: 2px;
 }

-
-li p { line-height: 1.25em; }
-
-.normal {
-    font-size: 1em;
+.remark-slide-number {
+    font-size: 12px;
 }

-.smaller {
-    font-size: .8em;
-}
-
-a, a > code {
-    color: #ff9300;
+a{
    text-decoration: none;
 }

-.code > pre, .code > code {
-    white-space: pre-wrap;
+td, th {
+    text-align: left;
+    padding-left: 5px;
+    padding-right: 5px;
+    border-bottom: 1px solid #ddd;
 }
-.remark-code, .remark-inline-code {
-    font-family: 'Consolas';
-    color: #ff9300;
-    font-size: 1.1em;
+
+
+.left-column5 {
+    width: 5%;
+    float: left;
 }
-.slightly-smaller {
-    font-size: .9em;
+
+.right-column95 {
+    width: 93%;
+    float: right;
 }
-    
-.remark-code-line-highlighted     {
-    background-color: #373832;
+
+
+.left-column95 {
+    width: 93%;
+    float: left;
 }
+
+.right-column5 {
+    width: 5%;
+    float: right;
+}
+
+.reduce90 {
+    font-size: 90%;
+}
+
+.reduce90 .remark-code{
+    font-size: 16px;
+}
+
+.reduce70 {
+    font-size: 70%;
+}
+
+.reduce70 .remark-code{
+    font-size: 12px;
+}
+
+.enlarge120 {
+    font-size: 120%;
+}
+
+.enlarge120 .remark-code{
+    font-size: 22px;
+}
+
+.strike {
+    text-decoration: line-through;
+}
+
 .pull-left {
    float: left;
    width: 47%;
@@ -105,106 +171,3 @@ a, a > code {
 .pull-right ~ p {
    clear: both;
 }
-#slideshow .slide .content code {
-}
-#slideshow .slide .content pre code {
-}
-
-.inverse {
-    background-color: #000;
-    background-repeat: no-repeat;
-    background-position: center;
-    background-size: contain;
-    color: #ffffff;
-}
-
-i, em, b, strong {
-    color: #ff9300;
-}
-
-.inverse h1, .inverse h2 {
-    color: #f3f3f3;
-    line-height: 0.8em;
-}
-
-.white {
-    background-color: #ffffff;
-    background-repeat: no-repeat;
-    background-position: center;
-    background-size: contain;
-    color: #000;
-}
-
-.strike {
-    text-decoration: line-through;
-}
-
-/* Slide-specific styling */
-#slide-inverse .footnote {
-    bottom: 12px;
-    left: 20px;
-}
-#slide-how .slides {
-    font-size: 0.9em;
-    position: absolute;
-    top:  151px;
-    right: 140px;
-}
-#slide-how .slides h3 {
-    margin-top: 0.2em;
-}
-#slide-how .slides .first, #slide-how .slides .second {
-    padding: 1px 20px;
-    height: 90px;
-    width: 120px;
-    -moz-box-shadow: 0 0 10px #777;
-    -webkit-box-shadow: 0 0 10px #777;
-    box-shadow: 0 0 10px #777;
-}
-#slide-how .slides .first {
-    background: #fff;
-    position: absolute;
-    top: 20%;
-    left: 20%;
-    z-index: 1;
-}
-#slide-how .slides .second {
-    position: relative;
-    background: #fff;
-    z-index: 0;
-}
-
-/* Two-column layout */
-.left-column {
-    color: #777;
-    width: 20%;
-    height: 92%;
-    float: left;
-}
-.left-column h2:last-of-type, .left-column h3:last-child {
-    color: #000;
-}
-.right-column {
-    width: 75%;
-    float: right;
-    padding-top: 1em;
-}
-
-/*First slide h1 orange, h2 white*/
-.special h1 {
-    color: #ff8800;
-}
-
-.special p {
-    color: #cbcbcb;
-}
-
-.special {
-    color: #ff8800;
-}
-
-/* Emphasis 2 */
-.special2 {
-    color: #00f900;
-}
-
@@ -11,18 +11,18 @@ subsequent row values are all numeric ! Will fail if any non numeric (eg '+' or
 ross lazarus for rgenetics
 august 20 2007
 """
-
 import logging
 import os
 import re
 import sys
-import urllib
 from cgi import escape

+from six.moves.urllib.parse import quote_plus
+
 from galaxy.datatypes import metadata
-from galaxy.datatypes.text import Html
 from galaxy.datatypes.metadata import MetadataElement
 from galaxy.datatypes.tabular import Tabular
+from galaxy.datatypes.text import Html
 from galaxy.util import nice_size
 from galaxy.web import url_for

@@ -95,9 +95,9 @@ class GenomeGraphs( Tabular ):
                                                   action='display_at',
                                                   filename='ucsc_' + site_name )
                    display_url = "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % (base_url, url_for( controller='root' ), dataset.id, type)
-                    display_url = urllib.quote_plus( display_url )
-                    # was display_url = urllib.quote_plus( "%s/display_as?id=%i&display_app=%s" % (base_url, dataset.id, type) )
-                    # redirect_url = urllib.quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, chrom, start, stop) )
+                    display_url = quote_plus( display_url )
+                    # was display_url = quote_plus( "%s/display_as?id=%i&display_app=%s" % (base_url, dataset.id, type) )
+                    # redirect_url = quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, chrom, start, stop) )
                    sl = ["%sdb=%s" % (site_url, dataset.dbkey ), ]
                    # sl.append("&hgt.customText=%s")
                    sl.append("&hgGenome_dataSetName=%s&hgGenome_dataSetDescription=%s" % (dataset.name, 'GalaxyGG_data'))
@@ -106,7 +106,7 @@ class GenomeGraphs( Tabular ):
                    sl.append("&hgGenome_doSubmitUpload=submit")
                    sl.append("&hgGenome_maxGapToFill=25000000&hgGenome_uploadFile=%s" % display_url)
                    s = ''.join(sl)
-                    s = urllib.quote_plus(s)
+                    s = quote_plus(s)
                    redirect_url = s
                    link = '%s?redirect_url=%s&display_url=%s' % ( internal_url, redirect_url, display_url )
                    ret_val.append( (site_name, link) )
@@ -117,17 +117,17 @@ class GenomeGraphs( Tabular ):
        Create HTML table, used for displaying peek
        """
        out = ['<table cellspacing="0" cellpadding="3">']
-        f = open(dataset.file_name, 'r')
-        d = f.readlines()[:5]
-        if len(d) == 0:
-            out = "Cannot find anything to parse in %s" % dataset.name
-            return out
-        hasheader = 0
-        try:
-            ['%f' % x for x in d[0][1:]]  # first is name - see if starts all numerics
-        except:
-            hasheader = 1
        try:
+            with open(dataset.file_name, 'r') as f:
+                d = f.readlines()[:5]
+            if len(d) == 0:
+                out = "Cannot find anything to parse in %s" % dataset.name
+                return out
+            hasheader = 0
+            try:
+                ['%f' % x for x in d[0][1:]]  # first is name - see if starts all numerics
+            except:
+                hasheader = 1
            # Generate column header
            out.append( '<tr>' )
            if hasheader:
@@ -150,16 +150,16 @@ class GenomeGraphs( Tabular ):
        Validate a gg file - all numeric after header row
        """
        errors = list()
-        infile = open(dataset.file_name, "r")
-        infile.next()  # header
-        for i, row in enumerate(infile):
-            ll = row.strip().split('\t')[1:]  # first is alpha feature identifier
-            badvals = []
-            for j, x in enumerate(ll):
-                try:
-                    x = float(x)
-                except:
-                    badvals.append('col%d:%s' % (j + 1, x))
+        with open(dataset.file_name, "r") as infile:
+            next(infile)  # header
+            for i, row in enumerate(infile):
+                ll = row.strip().split('\t')[1:]  # first is alpha feature identifier
+                badvals = []
+                for j, x in enumerate(ll):
+                    try:
+                        x = float(x)
+                    except:
+                        badvals.append('col%d:%s' % (j + 1, x))
        if len(badvals) > 0:
            errors.append('row %d, %s' % (' '.join(badvals)))
            return errors
@@ -219,7 +219,7 @@ class rgTabList(Tabular):

    def display_peek( self, dataset ):
        """Returns formated html of peek"""
-        return Tabular.make_html_table( self, dataset, column_names=self.column_names )
+        return self.make_html_table( dataset, column_names=self.column_names )

    def get_mime(self):
        """Returns the mime type of the datatype"""
@@ -246,8 +246,8 @@ class rgSampleList(rgTabList):
        # this is what Plink wants as at 2009

    def sniff(self, filename):
-        infile = open(filename, "r")
-        header = infile.next()  # header
+        with open(filename, "r") as infile:
+            header = next(infile)  # header
        if header[0] == 'FID' and header[1] == 'IID':
            return True
        else:
@@ -287,7 +287,7 @@ class Rgenetics(Html):
    def generate_primary_file( self, dataset=None ):
        rval = ['<html><head><title>Rgenetics Galaxy Composite Dataset </title></head><p/>']
        rval.append('<div>This composite dataset is composed of the following files:<p/><ul>')
-        for composite_name, composite_file in self.get_composite_files( dataset=dataset ).iteritems():
+        for composite_name, composite_file in self.get_composite_files( dataset=dataset ).items():
            fn = composite_name
            opt_text = ''
            if composite_file.optional:
@@ -617,7 +617,7 @@ class RexpBase( Html ):
            del useConc[i]  # get rid of concordance
            del useCols[i]  # and usecols entry
        for i, conc in enumerate(useConc):  # these are all unique columns for the design matrix
-                ccounts = sorted([(conc.get(code, 0), code) for code in conc.keys()])  # decorate
+                ccounts = sorted((conc.get(code, 0), code) for code in conc.keys())  # decorate
                cc = [(x[1], x[0]) for x in ccounts]  # list of code count tuples
                codeDetails = (head[useCols[i]], cc)  # ('foo',[('a',3),('b',11),..])
                listCol.append(codeDetails)
@@ -6,10 +6,10 @@ import math
 import os
 import sys
 import tempfile
-import urllib

 import numpy
 from bx.intervals.io import GenomicIntervalReader, ParseError
+from six.moves.urllib.parse import quote_plus

 from galaxy import util
 from galaxy.datatypes import metadata
@@ -19,8 +19,10 @@ from galaxy.datatypes.tabular import Tabular
 from galaxy.datatypes.util.gff_util import parse_gff_attributes
 from galaxy.web import url_for

-import data
-import dataproviders
+from . import (
+    data,
+    dataproviders
+)

 log = logging.getLogger(__name__)

@@ -86,7 +88,7 @@ class Interval( Tabular ):
                        self.init_meta( dataset )
                        line = line.strip( '#' )
                        elems = line.split( '\t' )
-                        for meta_name, header_list in alias_spec.iteritems():
+                        for meta_name, header_list in alias_spec.items():
                            for header_val in header_list:
                                if header_val in elems:
                                    # found highest priority header to meta_name
@@ -239,7 +241,7 @@ class Interval( Tabular ):

    def display_peek( self, dataset ):
        """Returns formated html of peek"""
-        return Tabular.make_html_table( self, dataset, column_parameter_alias={'chromCol': 'Chrom', 'startCol': 'Start', 'endCol': 'End', 'strandCol': 'Strand', 'nameCol': 'Name'} )
+        return self.make_html_table( dataset, column_parameter_alias={'chromCol': 'Chrom', 'startCol': 'Start', 'endCol': 'End', 'strandCol': 'Strand', 'nameCol': 'Name'} )

    def ucsc_links( self, dataset, type, app, base_url ):
        """
@@ -263,10 +265,10 @@ class Interval( Tabular ):
        for site_name, site_url in valid_sites:
            internal_url = url_for( controller='dataset', dataset_id=dataset.id,
                                    action='display_at', filename='ucsc_' + site_name )
-            display_url = urllib.quote_plus( "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at"
-                                             % (base_url, url_for( controller='root' ), dataset.id, type) )
-            redirect_url = urllib.quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s"
-                                              % (site_url, dataset.dbkey, chrom, start, stop ) )
+            display_url = quote_plus( "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" %
+                                      (base_url, url_for( controller='root' ), dataset.id, type) )
+            redirect_url = quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" %
+                                       (site_url, dataset.dbkey, chrom, start, stop ) )
            link = '%s?redirect_url=%s&display_url=%s' % ( internal_url, redirect_url, display_url )
            ret_val.append( ( site_name, link ) )
        return ret_val
@@ -286,7 +288,7 @@ class Interval( Tabular ):

        while True:
            try:
-                reader.next()
+                next(reader)
            except ParseError as e:
                errors.append(e)
            except StopIteration:
@@ -635,8 +637,8 @@ class _RemoteCallMixin:
        """
        internal_url = "%s" % url_for( controller='dataset', dataset_id=dataset.id, action='display_at', filename='%s_%s' % ( type, site_name ) )
        base_url = app.config.get( "display_at_callback", base_url )
-        display_url = urllib.quote_plus( "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" %
-                                         ( base_url, url_for( controller='root' ), dataset.id, type ) )
+        display_url = quote_plus( "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" %
+                                  ( base_url, url_for( controller='root' ), dataset.id, type ) )
        link = '%s?redirect_url=%s&display_url=%s' % ( internal_url, redirect_url, display_url )
        return link

@@ -723,7 +725,7 @@ class Gff( Tabular, _RemoteCallMixin ):

    def display_peek( self, dataset ):
        """Returns formated html of peek"""
-        return Tabular.make_html_table( self, dataset, column_names=self.column_names )
+        return self.make_html_table( dataset, column_names=self.column_names )

    def get_estimated_display_viewport( self, dataset ):
        """
@@ -808,7 +810,7 @@ class Gff( Tabular, _RemoteCallMixin ):
        if seqid is not None:
            for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build('ucsc', dataset.dbkey ):
                if site_name in app.datatypes_registry.get_display_sites('ucsc'):
-                    redirect_url = urllib.quote_plus(
+                    redirect_url = quote_plus(
                        "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" %
                        ( site_url, dataset.dbkey, seqid, start, stop ) )
                    link = self._get_remote_call_url( redirect_url, site_name, dataset, type, app, base_url )
@@ -823,7 +825,7 @@ class Gff( Tabular, _RemoteCallMixin ):
                if site_name in app.datatypes_registry.get_display_sites('gbrowse'):
                    if seqid.startswith( 'chr' ) and len( seqid ) > 3:
                        seqid = seqid[3:]
-                    redirect_url = urllib.quote_plus( "%s/?q=%s:%s..%s&eurl=%%s" % ( site_url, seqid, start, stop ) )
+                    redirect_url = quote_plus( "%s/?q=%s:%s..%s&eurl=%%s" % ( site_url, seqid, start, stop ) )
                    link = self._get_remote_call_url( redirect_url, site_name, dataset, type, app, base_url )
                    ret_val.append( ( site_name, link ) )
        return ret_val
@@ -1170,7 +1172,7 @@ class Wiggle( Tabular, _RemoteCallMixin ):
                if site_name in app.datatypes_registry.get_display_sites('gbrowse'):
                    if chrom.startswith( 'chr' ) and len( chrom ) > 3:
                        chrom = chrom[3:]
-                    redirect_url = urllib.quote_plus( "%s/?q=%s:%s..%s&eurl=%%s" % ( site_url, chrom, start, stop ) )
+                    redirect_url = quote_plus( "%s/?q=%s:%s..%s&eurl=%%s" % ( site_url, chrom, start, stop ) )
                    link = self._get_remote_call_url( redirect_url, site_name, dataset, type, app, base_url )
                    ret_val.append( ( site_name, link ) )
        return ret_val
@@ -1181,14 +1183,14 @@ class Wiggle( Tabular, _RemoteCallMixin ):
        if chrom is not None:
            for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build('ucsc', dataset.dbkey ):
                if site_name in app.datatypes_registry.get_display_sites('ucsc'):
-                    redirect_url = urllib.quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % ( site_url, dataset.dbkey, chrom, start, stop ) )
+                    redirect_url = quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % ( site_url, dataset.dbkey, chrom, start, stop ) )
                    link = self._get_remote_call_url( redirect_url, site_name, dataset, type, app, base_url )
                    ret_val.append( ( site_name, link ) )
        return ret_val

    def display_peek( self, dataset ):
        """Returns formated html of peek"""
-        return Tabular.make_html_table( self, dataset, skipchars=['track', '#'] )
+        return self.make_html_table( dataset, skipchars=['track', '#'] )

    def set_meta( self, dataset, overwrite=True, **kwd ):
        max_data_lines = None
@@ -1266,7 +1268,7 @@ class Wiggle( Tabular, _RemoteCallMixin ):
        x = numpy.arange( t_start, t_end ) * resolution
        y = data[ t_start : t_end ]

-        return zip(x.tolist(), y.tolist())
+        return list(zip(x.tolist(), y.tolist()))

    def get_track_resolution( self, dataset, start, end):
        range = end - start
@@ -1305,7 +1307,7 @@ class CustomTrack ( Tabular ):

    def display_peek( self, dataset ):
        """Returns formated html of peek"""
-        return Tabular.make_html_table( self, dataset, skipchars=['track', '#'] )
+        return self.make_html_table( dataset, skipchars=['track', '#'] )

    def get_estimated_display_viewport( self, dataset, chrom_col=None, start_col=None, end_col=None ):
        """Return a chrom, start, stop tuple for viewing a file."""
@@ -1372,8 +1374,8 @@ class CustomTrack ( Tabular ):
            for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build('ucsc', dataset.dbkey):
                if site_name in app.datatypes_registry.get_display_sites('ucsc'):
                    internal_url = "%s" % url_for( controller='dataset', dataset_id=dataset.id, action='display_at', filename='ucsc_' + site_name )
-                    display_url = urllib.quote_plus( "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % (base_url, url_for( controller='root' ), dataset.id, type) )
-                    redirect_url = urllib.quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, chrom, start, stop ) )
+                    display_url = quote_plus( "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % (base_url, url_for( controller='root' ), dataset.id, type) )
+                    redirect_url = quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, chrom, start, stop ) )
                    link = '%s?redirect_url=%s&display_url=%s' % ( internal_url, redirect_url, display_url )
                    ret_val.append( (site_name, link) )
        return ret_val
@@ -1519,6 +1521,14 @@ class ScIdx(Tabular):
            fh = open(filename, "r")
            while True:
                line = fh.readline()
+                if not line:
+                    # EOF
+                    if count > 1:
+                        # The second line is always the labels:
+                        # chrom index forward reverse value
+                        # We need at least the column labels and a data line.
+                        return True
+                    return False
                line = line.strip()
                # The first line is always a comment like this:
                # 2015-11-23 20:18:56.51;input.bam;READ1
@@ -1528,14 +1538,6 @@ class ScIdx(Tabular):
                        continue
                    else:
                        return False
-                if not line:
-                    # EOF
-                    if count > 1:
-                        # The second line is always the labels:
-                        # chrom index forward reverse value
-                        # We need at least the column labels and a data line.
-                        return True
-                    return False
                # Skip first line.
                if count > 1:
                    items = line.split('\t')
@@ -61,12 +61,12 @@ class PepXmlReport(Tabular):
    file_ext = "pepxml.tsv"

    def __init__(self, **kwd):
-        Tabular.__init__(self, **kwd)
+        super(PepXmlReport, self).__init__(**kwd)
        self.column_names = ['Protein', 'Peptide', 'Assumed Charge', 'Neutral Pep Mass (calculated)', 'Neutral Mass', 'Retention Time', 'Start Scan', 'End Scan', 'Search Engine', 'PeptideProphet Probability', 'Interprophet Probabaility']

    def display_peek(self, dataset):
        """Returns formated html of peek"""
-        return Tabular.make_html_table(self, dataset, column_names=self.column_names)
+        return self.make_html_table(dataset, column_names=self.column_names)


 class ProtXmlReport(Tabular):
@@ -76,7 +76,7 @@ class ProtXmlReport(Tabular):
    comment_lines = 1

    def __init__(self, **kwd):
-        Tabular.__init__(self, **kwd)
+        super(ProtXmlReport, self).__init__(**kwd)
        self.column_names = [
            "Entry Number", "Group Probability",
            "Protein", "Protein Link", "Protein Probability",
@@ -91,7 +91,7 @@ class ProtXmlReport(Tabular):

    def display_peek(self, dataset):
        """Returns formated html of peek"""
-        return Tabular.make_html_table(self, dataset, column_names=self.column_names)
+        return self.make_html_table(dataset, column_names=self.column_names)


 class ProteomicsXml(GenericXml):
@@ -407,7 +407,7 @@ class Taxonomy( Tabular ):

    def display_peek( self, dataset ):
        """Returns formated html of peek"""
-        return super(Taxonomy, self).make_html_table( dataset, column_names=self.column_names )
+        return self.make_html_table( dataset, column_names=self.column_names )


@dataproviders.decorators.has_dataproviders
@@ -427,7 +427,7 @@ class Sam( Tabular ):

    def display_peek( self, dataset ):
        """Returns formated html of peek"""
-        return super( Sam, self ).make_html_table( dataset, column_names=self.column_names )
+        return self.make_html_table( dataset, column_names=self.column_names )

    def sniff( self, filename ):
        """
@@ -613,7 +613,7 @@ class Pileup( Tabular ):

    def display_peek( self, dataset ):
        """Returns formated html of peek"""
-        return super( Pileup, self ).make_html_table( dataset, column_parameter_alias={'chromCol': 'Chrom', 'startCol': 'Start', 'baseCol': 'Base'} )
+        return self.make_html_table( dataset, column_parameter_alias={'chromCol': 'Chrom', 'startCol': 'Start', 'baseCol': 'Base'} )

    def repair_methods( self, dataset ):
        """Return options for removing errors along with a description"""
@@ -690,7 +690,7 @@ class Vcf( Tabular ):

    def display_peek( self, dataset ):
        """Returns formated html of peek"""
-        return super( Vcf, self ).make_html_table( dataset, column_names=self.column_names )
+        return self.make_html_table( dataset, column_names=self.column_names )

    def set_meta( self, dataset, **kwd ):
        super( Vcf, self ).set_meta( dataset, **kwd )
@@ -800,7 +800,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
        replace the old tool.
        """
        if tool_id not in self._tools_by_id:
-            message = "No tool with id %s" % escape( tool_id )
+            message = "No tool with id '%s'." % escape( tool_id )
            status = 'error'
        else:
            old_tool = self._tools_by_id[ tool_id ]
@@ -827,10 +827,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
            # (Re-)Register the reloaded tool, this will handle
            #  _tools_by_id and _tool_versions_by_id
            self.register_tool( new_tool )
-            message = "Reloaded the tool:<br/>"
-            message += "<b>name:</b> %s<br/>" % escape( old_tool.name )
-            message += "<b>id:</b> %s<br/>" % escape( old_tool.id )
-            message += "<b>version:</b> %s" % escape( old_tool.version )
+            message = { 'name' : old_tool.name, 'id' : old_tool.id, 'version' : old_tool.version }
            status = 'done'
        return message, status

@@ -4888,16 +4888,17 @@ for more examples.
 ]]></xs:documentation>
    </xs:annotation>
    <xs:sequence/>
-    <xs:attribute name="input" type="xs:string" use="required">
+    <xs:attribute name="input" type="xs:string">
      <xs:annotation>
-        <xs:documentation xml:lang="en">This value must be the attribute name of
+        <xs:documentation xml:lang="en">This attribute should be the name of
 the desired input parameter (e.g. ``input="out_format"`` above).</xs:documentation>
      </xs:annotation>
    </xs:attribute>
    <xs:attribute name="value" type="xs:string" use="required">
      <xs:annotation>
-        <xs:documentation xml:lang="en">This value must also be an attribute
-name of an input parameter (e.g. ``value="interval"`` above).</xs:documentation>
+        <xs:documentation xml:lang="en">This must be a possible value of the ``input``
+parameter (e.g. ``value="interval"`` above), or of the deprecated ``input_dataset``'s
+``attribute.</xs:documentation>
      </xs:annotation>
    </xs:attribute>
    <xs:attribute name="format" type="xs:string" use="required">
@@ -4908,6 +4909,16 @@ name of an input parameter (e.g. ``value="interval"`` above).</xs:documentation>
 for a list of supported formats.</xs:documentation>
      </xs:annotation>
    </xs:attribute>
+    <xs:attribute name="input_dataset" type="xs:string" gxdocs:deprecated="true">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Deprecated.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="attribute" type="xs:string" gxdocs:deprecated="true">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Deprecated.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
  </xs:complexType>

  <xs:complexType name="Citations">
@@ -99,7 +99,7 @@ class Admin( object ):
        if params.get( 'reload_tool_button', False ):
            tool_id = kwd.get( 'tool_id', None )
            galaxy.queue_worker.send_control_task(trans.app, 'reload_tool', noop_self=True, kwargs={'tool_id': tool_id} )
-            message, status = trans.app.toolbox.reload_tool_by_id( tool_id)
+            message, status = trans.app.toolbox.reload_tool_by_id( tool_id )
        return trans.fill_template( '/admin/reload_tool.mako',
                                    tool_id=tool_id,
                                    toolbox=toolbox,
@@ -362,7 +362,7 @@ class GalaxyWebTransaction( base.DefaultWebTransaction,
        """
        Authenticate for the API via key or session (if available).
        """
-        api_key = self.request.params.get('key', None)
+        api_key = self.request.params.get('key', None) or self.request.headers.get( 'x-api-key', None )
        secure_id = self.get_cookie( name=session_cookie )
        api_key_supplied = self.environ.get('is_api_request', False) and api_key
        if api_key_supplied and self._check_master_api_key( api_key ):
@@ -106,7 +106,9 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ):
        """
        galaxy.queue_worker.send_control_task( trans.app, 'reload_tool', noop_self=True, kwargs={ 'tool_id': id } )
        message, status = trans.app.toolbox.reload_tool_by_id( id )
-        return { status: message }
+        if status == 'error':
+            raise exceptions.MessageException( message )
+        return { 'message': message }

    @expose_api
    @web.require_admin
@@ -15,10 +15,12 @@ def attach_ordered_steps( workflow, steps ):
    fails - the workflow contains cycles so it mark it as such.
    """
    ordered_steps = order_workflow_steps( steps )
-    workflow.has_cycles = not bool( ordered_steps )
-    for i, step in enumerate( ordered_steps or steps ):
+    workflow.has_cycles = True
+    if ordered_steps:
+        workflow.has_cycles = False
+        workflow.steps = ordered_steps
+    for i, step in enumerate( workflow.steps ):
        step.order_index = i
-        workflow.steps.append( step )
    return workflow.has_cycles


@@ -9,10 +9,11 @@
    <textarea id="source">
 ${content}
    </textarea>
-    <script src="remark-latest.min.js">
+    <script src="https://remarkjs.com/downloads/remark-latest.min.js" type="text/javascript">
    </script>
-    <script>
-      var slideshow = remark.create();
+    <script type="text/javascript">
+      var slideshow = remark.create({navigation: {scroll: false,}});
+      var hljs = remark.highlighter.engine;
    </script>
  </body>
 </html>
@@ -1,8 +1,9 @@
 <%inherit file="/base.mako"/>
 <%namespace file="/message.mako" import="render_msg" />
 <%
-   from galaxy.tools import Tool
-   from galaxy.tools.toolbox import ToolSection
+    from cgi import escape
+    from galaxy.tools import Tool
+    from galaxy.tools.toolbox import ToolSection
 %>

 <script type="text/javascript">
@@ -24,9 +25,14 @@ $().ready(function() {
 });
 </script>

-%if message:
-    ${render_msg( message, status )}
-%endif
+<%
+    if message:
+        html_message = "Reloaded the tool:<br/>"
+        html_message += "<b>name:</b> %s<br/>" % escape( message[ 'name' ] )
+        html_message += "<b>id:</b> %s<br/>" % escape( message[ 'id' ] )
+        html_message += "<b>version:</b> %s" % escape( message[ 'version' ] )
+        render_msg( html_message, status )
+%>

 <div class="toolForm">
    <div class="toolFormTitle">Reload Tool</div>
@@ -912,7 +912,7 @@ class ToolsTestCase( api.ApiTestCase ):
        self.assertEquals( len( outputs ), 2 )
        output1 = outputs[ 0 ]
        output2 = outputs[ 1 ]
-        self.dataset_populator.wait_for_history( history_id, timeout=25 )
+        self.dataset_populator.wait_for_history( history_id )
        output1_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output1 )
        output2_content = self.dataset_populator.get_history_dataset_content( history_id, dataset=output2 )
        self.assertEquals( output1_content.strip(), "123\n789" )
@@ -98,7 +98,7 @@ class WorkflowExtractionApiTestCase( BaseWorkflowsApiTestCase ):
            history_id=self.history_id,
        )
        job_id2 = reduction_run_output[ "jobs" ][ 0 ][ "id" ]
-        self.dataset_populator.wait_for_history( self.history_id, assert_ok=True, timeout=20 )
+        self.dataset_populator.wait_for_history( self.history_id, assert_ok=True )
        downloaded_workflow = self._extract_and_download_workflow(
            dataset_collection_ids=[ hdca[ "hid" ] ],
            job_ids=[ job_id1, job_id2 ],
@@ -425,7 +425,7 @@ test_data:
        )
        implicit_hdca = run_output1[ "implicit_collections" ][ 0 ]
        job_id = run_output1[ "jobs" ][ 0 ][ "id" ]
-        self.dataset_populator.wait_for_history( history_id, assert_ok=True, timeout=20 )
+        self.dataset_populator.wait_for_history( history_id, assert_ok=True )
        return implicit_hdca, job_id

    def __check_workflow(
@@ -17,7 +17,7 @@ workflow_str = resource_string( __name__, "data/test_workflow_1.ga" )
 workflow_random_x2_str = resource_string( __name__, "data/test_workflow_2.ga" )


-DEFAULT_TIMEOUT = 15  # Secs to wait for state to turn ok
+DEFAULT_TIMEOUT = 60  # Secs to wait for state to turn ok


 def skip_without_tool( tool_id ):
@@ -0,0 +1,30 @@
+<tool id="output_format_deprecated_when" name="output_format_deprecated_when" version="1.0.0">
+  <command>
+    echo "test" > 5;
+  </command>
+  <inputs>
+    <param name="input_data_1" type="data" format="data" label="input_data_1" />
+  </inputs>
+  <outputs>
+    <data format="txt" from_work_dir="5" name="change_format_deprecated">
+      <change_format>
+        <when input_dataset="input_data_1" attribute="extension" value="fastq" format="fastq" />
+      </change_format>
+    </data>
+  </outputs>
+  <tests>
+    <test>
+      <param name="input_data_1" value="1.fastqsanger" ftype="fastq" />
+      <output name="change_format_deprecated" ftype="fastq">
+        <assert_contents><has_line line="test" /></assert_contents>
+      </output>
+    </test>
+    <test>
+      <param name="input_data_1" value="1.fastqsanger" ftype="fastqsanger" />
+      <output name="change_format_deprecated" ftype="txt">
+        <!-- Fails to check subclasses, use format_soure if possible -->
+        <assert_contents><has_line line="test" /></assert_contents>
+      </output>
+    </test>
+  </tests>
+</tool>
@@ -48,6 +48,7 @@
  <tool file="gzipped_inputs.xml" />
  <tool file="output_order.xml" />
  <tool file="output_format.xml" />
+  <tool file="output_format_deprecated_when.xml" />
  <tool file="output_format_collection.xml" />
  <tool file="output_filter.xml" />
  <tool file="output_filter_exception_1.xml" />
@@ -33,7 +33,7 @@ from base.workflows_format_2 import (

 from galaxy.util import asbool

-DEFAULT_WAIT_TIMEOUT = 15
+DEFAULT_WAIT_TIMEOUT = 60
 DEFAULT_TEST_ERRORS_DIRECTORY = os.path.abspath("database/test_errors")
 DEFAULT_SELENIUM_BROWSER = "auto"
 DEFAULT_SELENIUM_REMOTE = False
@@ -10,16 +10,24 @@ STOCK_TOURS_DIRECTORY = os.path.join(galaxy_root_path, "config", "plugins", "tou

 class TestStockToursTestCase(SeleniumTestCase):

-    @selenium_test
-    def test_core_galaxy_ui(self):
-        sleep_on_steps = {
-            "Tools": 20,  # Give upload a chance to take so tool form is filled in.
-            "History": 20,
-        }
-        self.run_tour(
-            os.path.join(STOCK_TOURS_DIRECTORY, "core.galaxy_ui.yaml"),
-            sleep_on_steps=sleep_on_steps,
-        )
+    # Test doesn't pass consistently on Jenkins yet, something is wrong is tool panel
+    # interactions. Example problems:
+    # - https://jenkins.galaxyproject.org/view/All/job/selenium/86/testReport/junit/selenium_tests.test_stock_tours/TestStockToursTestCase/test_core_galaxy_ui/
+    # - https://jenkins.galaxyproject.org/view/All/job/selenium/83/testReport/junit/selenium_tests.test_stock_tours/TestStockToursTestCase/test_core_galaxy_ui/
+    # - https://jenkins.galaxyproject.org/view/All/job/selenium/81/testReport/junit/selenium_tests.test_stock_tours/TestStockToursTestCase/test_core_galaxy_ui/
+    # I'd think that just pausing a bit between transitions to allow the tool panel to
+    # settle would fix it but it doesn't seem to in my initial testing. -John
+    # Tracking with https://github.com/galaxyproject/galaxy/issues/3598
+    # @selenium_test
+    # def test_core_galaxy_ui(self):
+    #    sleep_on_steps = {
+    #        "Tools": 20,  # Give upload a chance to take so tool form is filled in.
+    #        "History": 20,
+    #    }
+    #    self.run_tour(
+    #        os.path.join(STOCK_TOURS_DIRECTORY, "core.galaxy_ui.yaml"),
+    #        sleep_on_steps=sleep_on_steps,
+    #    )

    @selenium_test
    def test_core_scratchbook(self):
@@ -12,9 +12,11 @@ class UploadsTestCase(SeleniumTestCase):

        histories = self.api_get("histories")
        current_id = histories[0]["id"]
+        self.history_panel_wait_for_hid_ok(1)

        history_contents = self.api_get("histories/%s/contents" % current_id)
-        assert len(history_contents) == 1
+        history_count = len(history_contents)
+        assert history_count == 1, "Incorrect number of items in history - expected 1, found %d" % history_count

        hda = history_contents[0]
        assert hda["name"] == '1.sam'