From 261eb7fe0f7af0c8a21d36f0afae304557ad6e8c Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 8 Mar 2019 12:58:01 -0500
Subject: [PATCH 001/114] Add RealTimeTools. Currently working for docker in
 local runner.

---
 .../scripts/entry/analysis/AnalysisRouter.js  |  12 +-
 client/galaxy/scripts/layout/menu.js          |   6 +
 client/galaxy/scripts/mvc/tool/tool-form.js   |  20 +-
 config/datatypes_conf.xml.sample              |   1 +
 config/galaxy.yml.realtime                    |  32 +++
 lib/galaxy/app.py                             |   3 +
 lib/galaxy/config.py                          |   2 +
 lib/galaxy/datatypes/realtime.py              |  76 +++++++
 lib/galaxy/jobs/__init__.py                   |   6 +
 lib/galaxy/jobs/runners/__init__.py           |  16 +-
 lib/galaxy/jobs/runners/local.py              |  87 ++++++++
 lib/galaxy/managers/realtime.py               | 202 ++++++++++++++++++
 lib/galaxy/model/__init__.py                  |  50 +++++
 lib/galaxy/model/mapping.py                   |  50 +++++
 .../migrate/versions/0147_realtime_tool.py    |  89 ++++++++
 lib/galaxy/tools/__init__.py                  |  30 ++-
 lib/galaxy/tools/actions/realtime.py          |  15 ++
 lib/galaxy/tools/deps/container_classes.py    |  21 +-
 lib/galaxy/tools/deps/dependencies.py         |   3 +-
 lib/galaxy/tools/deps/docker_util.py          |  29 +++
 lib/galaxy/tools/deps/singularity_util.py     |   2 +
 lib/galaxy/tools/execute.py                   |   9 +
 lib/galaxy/tools/parser/interface.py          |   5 +
 lib/galaxy/tools/parser/xml.py                |  15 +-
 .../realtime/rpc/key_type_token_mapping.py    |  31 +++
 lib/galaxy/webapps/galaxy/api/tools.py        |   1 +
 .../webapps/galaxy/controllers/realtime.py    | 148 +++++++++++++
 .../webapps/galaxy/controllers/tool_runner.py |   3 +-
 28 files changed, 951 insertions(+), 13 deletions(-)
 create mode 100644 config/galaxy.yml.realtime
 create mode 100644 lib/galaxy/datatypes/realtime.py
 create mode 100644 lib/galaxy/managers/realtime.py
 create mode 100644 lib/galaxy/model/migrate/versions/0147_realtime_tool.py
 create mode 100644 lib/galaxy/tools/actions/realtime.py
 create mode 100644 lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
 create mode 100644 lib/galaxy/webapps/galaxy/controllers/realtime.py

diff --git a/client/galaxy/scripts/entry/analysis/AnalysisRouter.js b/client/galaxy/scripts/entry/analysis/AnalysisRouter.js
index 2d8b1eb123e..80345e39ae8 100644
--- a/client/galaxy/scripts/entry/analysis/AnalysisRouter.js
+++ b/client/galaxy/scripts/entry/analysis/AnalysisRouter.js
@@ -72,7 +72,8 @@ export const getAnalysisRouter = Galaxy =>
             "(/)datasets(/)list(/)": "show_datasets",
             "(/)custom_builds": "show_custom_builds",
             "(/)datasets/edit": "show_dataset_edit_attributes",
-            "(/)datasets/error": "show_dataset_error"
+            "(/)datasets/error": "show_dataset_error",
+            "(/)realtime/list": "show_realtime_list"
         },
 
         require_login: ["show_user", "show_user_form", "show_workflows"],
@@ -109,6 +110,15 @@ export const getAnalysisRouter = Galaxy =>
             this.page.display(new FormWrapper.View(_.extend(model.get(form_id), { active_tab: "user" })));
         },
 
+        show_realtime_list: function() {
+            this.page.display(
+                new GridView({
+                    url_base: `${getAppRoot()}realtime/list`,
+                    active_tab: "analysis"
+                })
+            );
+        },
+
         show_visualizations: function(action_id) {
             var activeTab = action_id == "list_published" ? "shared" : "user";
             this.page.display(
diff --git a/client/galaxy/scripts/layout/menu.js b/client/galaxy/scripts/layout/menu.js
index 286e4150d99..d32e1ab0b29 100644
--- a/client/galaxy/scripts/layout/menu.js
+++ b/client/galaxy/scripts/layout/menu.js
@@ -310,6 +310,12 @@ var Collection = Backbone.Collection.extend({
                     {
                         title: _l("Saved Visualizations"),
                         url: "visualizations/list",
+                        target: "__use_router__",
+                        divider: true,
+                    },
+                    {
+                        title: _l("Active RealTimeTools"),
+                        url: "realtime/list",
                         target: "__use_router__"
                     }
                 ]
diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js
index f5e11bfeaed..94f108e60b4 100644
--- a/client/galaxy/scripts/mvc/tool/tool-form.js
+++ b/client/galaxy/scripts/mvc/tool/tool-form.js
@@ -346,6 +346,22 @@ var View = Backbone.View.extend({
         return inputs;
     },
 
+    _getViewResult: function(response) {
+        if (response.view_result) {
+            return `<div class="infomessagelarge">
+                        <p>
+                            <a href="${response.view_result}">
+                            There is a RealTimeTool result view available, click here to display. You can also click on the display eye icon (<span class="fa fa-eye"></span>)
+                            associated with the RealTimeTool Dataset to view.</a>
+                        </p>
+                        <p>
+                            You may also access all active RealTimeTools from the User menu.
+                        </p>
+                    </div>`;
+        }
+        return "";
+    },
+
     _templateRow: function(list, title, max = 3) {
         var blurb = "";
         list.sort(function(a, b) {
@@ -376,7 +392,9 @@ var View = Backbone.View.extend({
             var ninputsText = ninputs > 1 ? `${ninputs} inputs` : `this input`;
             var noutputsText = noutputs > 1 ? `${noutputs} outputs` : `this output`;
             var tool_name = this.form.model.get("name");
-            return `<div class="donemessagelarge">
+            var view_result = this._getViewResult(response);
+            return `${view_result}
+                    <div class="donemessagelarge">
                         <p>
                             Executed <b>${tool_name}</b> and successfully added ${njobsText} to the queue.
                         </p>
diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample
index 61d4345f7df..2b068fe2d8c 100644
--- a/config/datatypes_conf.xml.sample
+++ b/config/datatypes_conf.xml.sample
@@ -170,6 +170,7 @@
       <display file="rviewer/bed.xml" inherit="true"/>
       <display file="igv/interval_as_bed.xml" inherit="true"/>
     </datatype>
+    <datatype extension="realtimetool" type="galaxy.datatypes.realtime:Realtime" description="Realtime Tool Output" mimetype="text/html" display_in_upload="false"/>
 
     <!-- ISA data types -->
     <datatype extension="isa-tab" type="galaxy.datatypes.isa:IsaTab" mimetype="application/isa-tools" display_in_upload="true" description="ISA-Tab data type." description_url="https://isa-tools.org"/>
diff --git a/config/galaxy.yml.realtime b/config/galaxy.yml.realtime
new file mode 100644
index 00000000000..25eeaa589ff
--- /dev/null
+++ b/config/galaxy.yml.realtime
@@ -0,0 +1,32 @@
+uwsgi:
+
+  http: localhost:8080
+
+  threads: 8
+
+  http-raw-body: True
+
+  offload-threads: 8
+
+  master: true
+
+  module: galaxy.webapps.galaxy.buildapp:uwsgi_app()
+
+  sessions: database/session_map.sqlite
+  python-raw: lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
+  route-run: log:dynamic_proxy_prefix ${dynamic_proxy_prefix}
+  route: ^/realtime-proxy/access/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)(()|(/.*)*)$ goto:realtime
+  route-run: goto:endendend
+  route-label: realtime
+  route: ^/realtime-proxy/access/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)(()|/.*)$ rpcvar:TARGET_HOST rtt_key_type_token_mapper $2 $1 $3 $4 $0
+  route: ^/realtime-proxy/access/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)((()|/.*)*)$ rewrite:$4
+  route-if: empty:${PATH_INFO} addvar:PATH_INFO=/
+  route-run: seturi:${PATH_INFO}
+  route-if-not: empty:${QUERY_STRING} seturi:${PATH_INFO}?${QUERY_STRING}
+  route-if-not: empty:${TARGET_HOST} httpdumb:${TARGET_HOST}
+  route-label: endendend
+
+
+galaxy:
+
+  realtime_prefix: /realtime-proxy/access
diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py
index 85b854cb339..a252b41bba0 100644
--- a/lib/galaxy/app.py
+++ b/lib/galaxy/app.py
@@ -16,6 +16,7 @@ from galaxy.managers.collections import DatasetCollectionManager
 from galaxy.managers.folders import FolderManager
 from galaxy.managers.histories import HistoryManager
 from galaxy.managers.libraries import LibraryManager
+from galaxy.managers.realtime import RealTimeManager
 from galaxy.managers.tags import GalaxyTagManager
 from galaxy.queue_worker import GalaxyQueueWorker
 from galaxy.tools.cache import (
@@ -216,6 +217,8 @@ class UniverseApplication(config.ConfiguresGalaxyMixin):
                 containers_conf=self.config.containers_conf
             )
 
+        self.realtime_manager = RealTimeManager(self)
+
         # Configure handling of signals
         handlers = {}
         if self.heartbeat:
diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py
index 43e3afff0ca..7c6fb75b1f1 100644
--- a/lib/galaxy/config.py
+++ b/lib/galaxy/config.py
@@ -689,6 +689,8 @@ class Configuration(object):
         self.dynamic_proxy_golang_docker_address = kwargs.get("dynamic_proxy_golang_docker_address", "unix:///var/run/docker.sock")
         self.dynamic_proxy_golang_api_key = kwargs.get("dynamic_proxy_golang_api_key", None)
 
+        self.realtime_prefix = kwargs.get("realtime_prefix", "/realtime-proxy/access")
+
         # Default chunk size for chunkable datatypes -- 64k
         self.display_chunk_size = int(kwargs.get('display_chunk_size', 65536))
 
diff --git a/lib/galaxy/datatypes/realtime.py b/lib/galaxy/datatypes/realtime.py
new file mode 100644
index 00000000000..b9ceb5e1bd8
--- /dev/null
+++ b/lib/galaxy/datatypes/realtime.py
@@ -0,0 +1,76 @@
+"""Binary classes"""
+from __future__ import print_function
+
+import logging
+
+from text import Html
+from galaxy.web import url_for
+
+
+log = logging.getLogger(__name__)
+
+
+class Realtime(Html):
+    """Class describing a RealtimeTool Default output file"""
+    file_ext = "realtimetool"
+    composite_type = 'auto_primary_file'
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        dataset.blurb = "RealtimeTool"
+        dataset.peek = 'Finished RealTimeTool'
+
+
+    def display_peek(self, dataset):
+        try:
+            if not dataset.peek:
+                self.set_peek(dataset)
+            return dataset.peek
+        except Exception:
+            return "Realtime Tool file"
+
+    def generate_primary_file(self, dataset=None):
+        """
+        This is called only at upload to write the html file
+        cannot rename the datasets here - they come with the default unfortunately
+        """
+        defined_files = self.get_composite_files(dataset=dataset).items()
+        rval = ["<html><head><title>Files for Composite Dataset (%s)</title></head>" % (self.file_ext)]
+        if defined_files:
+            rval.append("<p/>This composite dataset is composed of the following defined files:<p/><ul>")
+            for composite_name, composite_file in defined_files:
+                opt_text = ''
+                if composite_file.optional:
+                    opt_text = ' (optional)'
+                missing_text = ''
+                if not os.path.exists(os.path.join(dataset.extra_files_path, composite_name)):
+                    missing_text = ' (missing)'
+                rval.append('<li><a href="%s">%s</a>%s%s</li>' % (composite_name, composite_name, opt_text, missing_text))
+            rval.append("</ul>")
+        defined_files = map(lambda x: x[0], defined_files)
+        extra_files = []
+        for (dirpath, dirnames, filenames) in os.walk(dataset.extra_files_path, followlinks=True):
+            for filename in filenames:
+                rel_path = os.path.relpath(os.path.join(dirpath, filename), dataset.extra_files_path)
+                if rel_path not in defined_files:
+                    extra_files.append(rel_path)
+        if extra_files:
+            rval.append("<p/>This composite dataset contains these undefined files:<p/><ul>")
+            for rel_path in extra_files:
+                rval.append('<li><a href="%s">%s</a></li>' % (rel_path, rel_path))
+            rval.append('</ul>')
+        if not (defined_files or extra_files):
+            rval.append("<p/>This composite dataset does not contain any files!<p/><ul>")
+        rval.append('</html>')
+        return "\n".join(rval)
+
+    def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, offset=None, ck_size=None, **kwd):
+        log.debug('dataset.realtime_tool.id %s', dataset.realtime_tool.id)
+        rtt = dataset.realtime_tool
+        if rtt and rtt.active:
+            return trans.response.send_redirect(url_for(controller='realtime', action='index', realtime_id=trans.security.encode_id(rtt.id)))
+        return super(Realtime, self).display_data(trans, dataset, preview=preview, filename=filename, to_ext=to_ext, offset=offset, ck_size=ck_size, **kwd)
+
+
+if __name__ == '__main__':
+    import doctest
+    doctest.testmod(sys.modules[__name__])
diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py
index ba0c4fa4330..d4f9f588a2b 100644
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -1891,6 +1891,12 @@ class JobWrapper(HasResourceParameters):
         for dataset in job.output_datasets:
             self.app.error_reports.default_error_plugin.submit_report(dataset, job, tool, user_submission=False)
 
+    def set_container(self, container):
+        if container:
+            cont = model.JobContainerAssociation( job=self.get_job(), container_type=container.container_type, container_name=container.container_name, container_info=container.container_info)
+            self.sa_session.add(cont)
+            self.sa_session.flush()
+
 
 class TaskWrapper(JobWrapper):
     """
diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py
index 4eb6ddaaf93..349fb78f5ad 100644
--- a/lib/galaxy/jobs/runners/__init__.py
+++ b/lib/galaxy/jobs/runners/__init__.py
@@ -260,7 +260,7 @@ class BaseJobRunner(object):
         container = self._find_container(job_wrapper)
         if not container and job_wrapper.requires_containerization:
             raise Exception("Failed to find a container when required, contact Galaxy admin.")
-        return build_command(
+        built=build_command(
             self,
             job_wrapper,
             include_metadata=include_metadata,
@@ -270,6 +270,11 @@ class BaseJobRunner(object):
             stdout_file=stdout_file,
             stderr_file=stderr_file,
         )
+        stored_container = job_wrapper.get_job().container
+        if container and stored_container:
+            if container.container_info:
+                stored_container.container_info=container.container_info
+        return built
 
     def get_work_dir_outputs(self, job_wrapper, job_working_directory=None, tool_working_directory=None):
         """
@@ -412,7 +417,9 @@ class BaseJobRunner(object):
             compute_tmp_directory = job_wrapper.tmp_directory()
 
         tool = job_wrapper.tool
-        tool_info = ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through)
+        # TODO: template tool ports?
+        guest_ports = [port.get('port') for port in tool.tool_ports]
+        tool_info = ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through, guest_ports=guest_ports)
         job_info = JobInfo(
             compute_working_directory,
             compute_tool_directory,
@@ -422,11 +429,14 @@ class BaseJobRunner(object):
         )
 
         destination_info = job_wrapper.job_destination.params
-        return self.app.container_finder.find_container(
+        container=self.app.container_finder.find_container(
             tool_info,
             destination_info,
             job_info
         )
+        if container:
+            container.set_name(job_wrapper=job_wrapper)
+        return container
 
     def _handle_runner_state(self, runner_state, job_state):
         try:
diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py
index 56f87b42ead..2337a0aeae8 100644
--- a/lib/galaxy/jobs/runners/local.py
+++ b/lib/galaxy/jobs/runners/local.py
@@ -3,6 +3,7 @@ Job runner plugin for executing jobs on the local system via the command line.
 """
 import datetime
 import errno
+import json
 import logging
 import os
 import subprocess
@@ -159,6 +160,15 @@ class LocalJobRunner(BaseJobRunner):
         # if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished
         job = job_wrapper.get_job()
         job_ext_output_metadata = job.get_external_output_metadata()
+        if job.container:
+            try:
+                self._stop_container(job_wrapper)
+            except Exception as e:
+                log.warning("stop_job(): %s: trying to stop container failed. (%s)" % (job.id, e))
+                try:
+                    self._kill_container(job_wrapper)
+                except Exception as e:
+                    log.warning("stop_job(): %s: trying to kill container failed. (%s)" % (job.id, e))
         try:
             pid = job_ext_output_metadata[0].job_runner_external_pid  # every JobExternalOutputMetadata has a pid set, we just need to take from one of them
             assert pid not in [None, '']
@@ -184,6 +194,7 @@ class LocalJobRunner(BaseJobRunner):
                 return
         else:
             log.warning("stop_job(): %s: PID %d refuses to die after signaling TERM/KILL" % (job.id, pid))
+        self.__kill_container(job_wrapper)
 
     def recover(self, job, job_wrapper):
         # local jobs can't be recovered
@@ -233,8 +244,83 @@ class LocalJobRunner(BaseJobRunner):
             os.killpg(proc.pid, 9)
         return proc.wait()  # reap
 
+    def _stop_container(self, job_wrapper):
+        return self._run_container_command(job_wrapper, 'stop')
+
+    def _kill_container(self, job_wrapper):
+        return self._run_container_command(job_wrapper, 'kill')
+
+    def _run_container_command(self, job_wrapper, command):
+        job = job_wrapper.get_job()
+        if job:
+            cont = job.container
+            if cont:
+                if cont.container_type == 'docker':
+                    exit_code = subprocess.call(cont.container_info['commands'][command],
+                                            shell=True,
+                                            env=self._environ,
+                                            preexec_fn=os.setpgrp)
+                    log.debug('docker container command exit code %s', exit_code)
+
+    def __handle_container(self, job_wrapper):
+        job = job_wrapper.get_job()
+        if job:
+            cont = job.container
+            if cont:
+                if cont.container_type == 'docker':
+                    # Do we need to do inspect, or is only ports ok?
+                    inspect=None
+                    for i in range(1, 11):
+                        with tempfile.TemporaryFile() as stdout_file:
+                            exit_code = subprocess.call(cont.container_info['commands']['inspect'],
+                                                    shell=True,
+                                                    stdout=stdout_file,
+                                                    env=self._environ,
+                                                    preexec_fn=os.setpgrp)
+                            if exit_code == 0:
+                                stdout_file.seek(0)
+                                inspect = stdout_file.read()
+                                break
+                        t=2*i
+                        log.debug("Container not found, sleeping for %s seconds.", t)
+                        sleep(t)
+                    cont.container_info['inspect'] = json.loads(inspect)
+                    rtt = job.realtime_tool
+                    if rtt:
+                        ports_raw=None
+                        for i in range(1, 11):
+                            with tempfile.TemporaryFile() as stdout_file:
+                                exit_code = subprocess.call(cont.container_info['commands']['port'],
+                                                        shell=True,
+                                                        stdout=stdout_file,
+                                                        #stderr=stderr_file,
+                                                        env=self._environ,
+                                                        preexec_fn=os.setpgrp)
+                                if exit_code == 0:
+                                    stdout_file.seek(0)
+                                    ports_raw = stdout_file.read()
+                                    break
+                            t=2*i
+                            log.debug("Container not found, sleeping for %s seconds.", t)
+                            sleep(t)
+
+                        for line in ports_raw.strip().split('\n'):
+                            #TODO: configure all ports at once, instead of individually
+                            #A port could e.g. be mapped multiple times, with diff entry URL
+                            tool, host =  line.split(" -> ", 1)
+                            hostname, port = host.split(':')
+                            tool_p, tool_prot = tool.split("/")
+                            ep = self.app.realtime_manager.configure_entry_point(rtt, tool_port=tool_p, host=hostname, port=port, protocol=tool_prot)
+                            if ep:
+                                # must add and flush the ep, here or elsewhere
+                                # otherwise, Galaxy's db doesn't get written to, but the sqlite db does
+                                job_wrapper.sa_session.add(ep)
+                                job_wrapper.sa_session.flush()
+
+
     def __poll_if_needed(self, proc, job_wrapper, job_id):
         # Only poll if needed (i.e. job limits are set)
+        self.__handle_container(job_wrapper)
         if not job_wrapper.has_limits():
             return
 
@@ -247,6 +333,7 @@ class LocalJobRunner(BaseJobRunner):
                 limit_state = job_wrapper.check_limits(runtime=datetime.datetime.now() - job_start)
                 if limit_state is not None:
                     job_wrapper.fail(limit_state[1])
+                    self.__kill_container(job_wrapper)
                     log.debug('(%s) Terminating process group' % job_id)
                     self._terminate(proc)
                     return True
diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
new file mode 100644
index 00000000000..317e88d37ab
--- /dev/null
+++ b/lib/galaxy/managers/realtime.py
@@ -0,0 +1,202 @@
+import logging
+import json
+import sqlite3
+
+
+from six import string_types
+from sqlalchemy import or_
+
+
+from galaxy import model
+
+from galaxy.util.filelock import FileLock
+
+
+log = logging.getLogger(__name__)
+
+REALTIMETOOL_DATASET_EXT = 'realtimetool'
+DATABASE_TABLE_NAME = 'gxrtproxy'
+
+
+class RealtimeSqlite(object):
+
+    def __init__(self, sqlite_filename, encode_id):
+        self.sqlite_filename = sqlite_filename
+        self.encode_id = encode_id
+
+    def get(self, key, key_type):
+        with FileLock(self.sqlite_filename):
+            conn = sqlite3.connect(self.sqlite_filename)
+            try:
+                c = conn.cursor()
+                select = '''SELECT token, host, port, info
+                            FROM %s
+                            WHERE key=? and key_type=?''' % (DATABASE_TABLE_NAME)
+                c.execute(select, (key,key_type,))
+                try:
+                    token, host, port, info = c.fetchone()
+                except TypeError:
+                    log.warning('get(): invalid key: %s key_type %s', key, key_type)
+                    return None
+                return dict(
+                    key=key,
+                    key_type=key_type,
+                    token=token,
+                    host=host,
+                    port=port,
+                    info=info)
+            finally:
+                conn.close()
+
+    def save(self, key, key_type, token, host, port, info=None):#, route_name):
+        """
+        Writeout a key, key_type, token, value store that is can be used for coordinating
+        with external resources.
+        """
+        assert key, ValueError("A non-zero length key is required.")
+        assert key_type, ValueError("A non-zero length key_type is required.")
+        assert token, ValueError("A non-zero length token is required.")
+        rval = None
+        with FileLock(self.sqlite_filename):
+            conn = sqlite3.connect(self.sqlite_filename)
+            try:
+                c = conn.cursor()
+                try:
+                    # Create table
+                    c.execute('''CREATE TABLE %s
+                                 (key text,
+                                  key_type text,
+                                  token text,
+                                  host text,
+                                  port integer,
+                                  info text,
+                                  PRIMARY KEY (key, key_type)
+                                  )'''%(DATABASE_TABLE_NAME))
+                except Exception:
+                    pass
+                delete = '''DELETE FROM %s WHERE key=? and key_type=?''' % (DATABASE_TABLE_NAME)
+                c.execute(delete, (key,key_type,))
+                insert = '''INSERT INTO %s
+                            (key, key_type, token, host, port, info)
+                            VALUES (?, ?, ?, ?, ?, ?)''' % (DATABASE_TABLE_NAME)
+                c.execute(insert,
+                          (key,
+                           key_type,
+                           token,
+                           host,
+                           port,
+                           info,
+                           ))
+                conn.commit()
+            finally:
+                conn.close()
+
+    def save_entry_point(self, entry_point):
+        """Convenience method to easily save an entry_point.
+        """
+        return self.save(self.encode_id(entry_point.id), entry_point.__class__.__name__, entry_point.token, entry_point.host, entry_point.port, None)
+
+
+
+
+class RealTimeManager(object):
+    """
+    Manager for dealing with RealTimeTools
+    """
+
+    def __init__(self, app):
+        self.model = app.model
+        self.security = app.security
+        self.propagator = RealtimeSqlite(app.config.proxy_session_map, app.security.encode_id) # Use existing db, but new table
+
+    #FIXME: Do we really want to directly associate a single dataset?
+    # Probably Not
+    def guess_associated_dataset(self, datasets):
+        for dataset in datasets:
+            dataset = dataset.dataset #AttributeError: 'JobToOutputDatasetAssociation' object has no attribute 'ext'
+            if dataset.ext == REALTIMETOOL_DATASET_EXT:
+                return dataset
+        return None
+    def create_entry_points(self, trans, rtt, tool, flush=True):
+        for entry in tool.tool_ports:
+            rtt_entry = self.model.RealTimeToolEntryPoint(realtime=rtt, tool_port=entry['port'], entry_url=entry['url'], name=entry['name'])
+            trans.sa_session.add(rtt_entry)
+        if flush:
+            trans.sa_session.flush()
+
+    def configure_entry_point(self, rtt, tool_port=None, host=None, port=None, protocol=None):
+        if tool_port is not None:
+            tool_port=int(tool_port)
+        not_configured = []
+        for entry in rtt.entry_points:
+            if entry.tool_port == tool_port:
+                entry.host=host
+                entry.port=port
+                entry.protocol=protocol
+                entry.configured = True
+                self.save_entry_point(entry)
+                return entry
+            elif not entry.configured:
+                not_configured.append(entry)
+
+        # TODO: port not found in entry, add to random one?
+        log.warning('tool port not found! %s', str(not_configured))
+        return None 
+        
+
+    def save_entry_point(self, entry_point):
+        """
+        Writeout a key, key_type, token, value store that is used validating access
+        """
+        self.propagator.save_entry_point(entry_point)
+   
+    def create_realtime(self, trans, job, tool):
+        # create from initial job
+        datasets = None
+        if job:
+            realtime_tool = self.model.RealTimeTool(job=job, user=trans.user, session=trans.galaxy_session, dataset=self.guess_associated_dataset(job.get_output_datasets()))
+            trans.sa_session.add(realtime_tool)
+            if tool:
+                self.create_entry_points(trans, realtime_tool, tool, flush=False)
+            trans.sa_session.flush()
+        else:
+            log.warning( 'Called RealTimeManager.create_realtime, but job is None' )
+
+    def get_nonterminal_for_user_by_trans(self, trans):
+        if trans.user:
+            jobs = trans.sa_session.query(trans.app.model.Job).filter(trans.app.model.Job.user == trans.user)
+        else:
+            jobs = trans.sa_session.query(trans.app.model.Job).filter(trans.app.model.Job.session_id == trans.get_galaxy_session().id)
+        def build_and_apply_filters(query, objects, filter_func):
+            if objects is not None:
+                if isinstance(objects, string_types):
+                    query = query.filter(filter_func(objects))
+                elif isinstance(objects, list):
+                    t = []
+                    for obj in objects:
+                        t.append(filter_func(obj))
+                    query = query.filter(or_(*t))
+            return query
+        jobs = build_and_apply_filters(jobs, trans.app.model.Job.non_ready_states, lambda s: trans.app.model.Job.state == s)
+        rtts = trans.sa_session.query(trans.app.model.RealTimeTool).filter(trans.app.model.RealTimeTool.job_id.in_([job.id for job in jobs]))
+        return trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).filter(trans.app.model.RealTimeToolEntryPoint.realtime_id.in_([ rtt.id for rtt in rtts ]))
+
+    def can_access_realtime(self, trans, realtime):
+        if realtime:
+            if trans.user is None:
+                galaxy_session = trans.get_galaxy_session()
+                if galaxy_session is None or realtime.session_id != galaxy_session.id:
+                    return False
+            elif realtime.user != trans.user:
+                return False
+        else:
+            return False
+        return True
+
+    def can_access_entry_point(self, trans, entry_point):
+        if entry_point:
+            return self.can_access_realtime(trans, entry_point.realtime)
+        return False
+
+    def stop(self, realtime):
+        raise Exception('not implemented')
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 10707240552..33873e75548 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -1299,6 +1299,56 @@ class JobImportHistoryArchive(RepresentById):
         self.archive_dir = archive_dir
 
 
+class JobContainerAssociation(RepresentById):
+    def __init__(self, job=None, container_type=None, container_name=None, container_info=None):
+        self.job = job
+        self.container_type = container_type
+        self.container_name = container_name
+        self.container_info = container_info or {}
+
+
+class RealTimeTool(RepresentById):
+    def __init__(self, job=None, user=None, session=None, dataset=None):
+        self.job = job
+        self.user = user
+        self.galaxy_session = session
+        self.dataset = dataset
+    @property
+    def active(self):
+        # FIXME: don't included queued?
+        return not self.job.finished
+    def to_dict(self, *args, **kwds):
+        return dict(job_id=self.job.id, user_id=self.user.id,galaxy_session_id=self.galaxy_session.id,dataset_id=self.dataset.id,active=self.active)#=self.realtime.to_dict(*args, **kwds))
+
+
+class RealTimeToolEntryPoint(RepresentById):
+    def __init__(self, realtime=None, name=None, token=None, tool_port=None, entry_url=None,
+                 host=None, port=None, protocol=None, configured=False):
+        self.realtime = realtime
+        self.name = name
+        if not token:
+            token = uuid4().get_hex()
+        self.token = token
+        self.tool_port = tool_port
+        self.entry_url = entry_url
+        self.host = host
+        self.port = port
+        self.protocol = protocol
+        self.configured = configured
+
+    def to_dict(self, *args, **kwds):
+        rval = dict(realtime_id=self.realtime.id)
+        for val in ['id', 'name','token','tool_port','entry_url','host','port','protocol','configured', 'created_time', 'modified_time', 'active']:
+            rval[val]=getattr(self, val)
+        return rval
+
+    @property
+    def active(self):
+        if self.configured:
+            return self.realtime.active
+        return False
+
+
 class GenomeIndexToolData(RepresentById):
     def __init__(self, job=None, params=None, dataset=None, deferred_job=None,
                  transfer_job=None, fasta_path=None, created_time=None, modified_time=None,
diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py
index 55df76e797e..500bc3e6dab 100644
--- a/lib/galaxy/model/mapping.py
+++ b/lib/galaxy/model/mapping.py
@@ -711,6 +711,41 @@ model.GenomeIndexToolData.table = Table(
     Column("indexer", String(64)),
     Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True))
 
+model.RealTimeTool.table = Table(
+    "realtime_tool", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("job_id", Integer, ForeignKey("job.id"), index=True),
+    Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True),
+    Column("session_id", Integer, ForeignKey("galaxy_session.id"), index=True),
+    Column("dataset_id", Integer, ForeignKey("history_dataset_association.id"), index=True),
+    Column("created_time", DateTime, default=now),
+    Column("modified_time", DateTime, default=now, onupdate=now))
+
+model.RealTimeToolEntryPoint.table = Table(
+    "realtime_tool_entry_point", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("realtime_id", Integer, ForeignKey("realtime_tool.id"), index=True),
+    Column("name", TEXT),
+    Column("token", TEXT),
+    Column("tool_port", Integer),
+    Column("entry_url", TEXT),
+    Column("host", TEXT),
+    Column("port", Integer),
+    Column("protocol", TEXT),
+    Column("configured", Boolean, default=False),
+    Column("created_time", DateTime, default=now),
+    Column("modified_time", DateTime, default=now, onupdate=now))
+
+model.JobContainerAssociation.table = Table(
+    "job_container_association", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("job_id", Integer, ForeignKey("job.id"), index=True),
+    Column("container_type", TEXT),
+    Column("container_name", TEXT),
+    Column("container_info", JSONType, nullable=True),
+    Column("created_time", DateTime, default=now),
+    Column("modified_time", DateTime, default=now, onupdate=now))
+
 model.Task.table = Table(
     "task", metadata,
     Column("id", Integer, primary_key=True),
@@ -2046,6 +2081,21 @@ mapper(model.GenomeIndexToolData, model.GenomeIndexToolData.table, properties=di
     transfer=relation(model.TransferJob, backref='transfer_job')
 ))
 
+mapper(model.RealTimeTool, model.RealTimeTool.table, properties=dict(
+    job=relation(model.Job, backref=backref('realtime_tool', uselist=False), uselist=False),
+    dataset=relation(model.HistoryDatasetAssociation, backref=backref('realtime_tool', uselist=False), uselist=False),
+    user=relation(model.User),
+    galaxy_session=relation(model.GalaxySession)
+))
+
+mapper(model.RealTimeToolEntryPoint, model.RealTimeToolEntryPoint.table, properties=dict(
+    realtime=relation(model.RealTimeTool, backref='entry_points', uselist=False)
+))
+
+mapper(model.JobContainerAssociation, model.JobContainerAssociation.table, properties=dict(
+    job=relation(model.Job, backref=backref('container', uselist=False), uselist=False)
+))
+
 mapper(model.PostJobAction, model.PostJobAction.table, properties=dict(
     workflow_step=relation(model.WorkflowStep,
         backref='post_job_actions',
diff --git a/lib/galaxy/model/migrate/versions/0147_realtime_tool.py b/lib/galaxy/model/migrate/versions/0147_realtime_tool.py
new file mode 100644
index 00000000000..15b2f535820
--- /dev/null
+++ b/lib/galaxy/model/migrate/versions/0147_realtime_tool.py
@@ -0,0 +1,89 @@
+"""
+Migration script to add a new tables for RealTimeTools.
+"""
+from __future__ import print_function
+
+import logging
+
+from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, String, Table, TEXT
+
+from galaxy.model.custom_types import JSONType
+from galaxy.model.orm.now import now
+
+log = logging.getLogger(__name__)
+metadata = MetaData()
+
+realtime_tool = Table(
+    "realtime_tool", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("job_id", Integer, ForeignKey("job.id"), index=True),
+    Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True),
+    Column("session_id", Integer, ForeignKey("galaxy_session.id"), index=True),
+    Column("dataset_id", Integer, ForeignKey("history_dataset_association.id"), index=True),
+    Column("created_time", DateTime, default=now),
+    Column("modified_time", DateTime, default=now, onupdate=now))
+
+
+realtime_tool_entry_point = Table(
+    "realtime_tool_entry_point", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("realtime_id", Integer, ForeignKey("realtime_tool.id"), index=True),
+    Column("name", TEXT),
+    Column("token", TEXT),
+    Column("tool_port", Integer),
+    Column("host", TEXT),
+    Column("port", Integer),
+    Column("protocol", TEXT),
+    Column("configured", Boolean, default=False),
+    Column("entry_url", TEXT),
+    Column("created_time", DateTime, default=now),
+    Column("modified_time", DateTime, default=now, onupdate=now))
+
+job_container_association = Table(
+    "job_container_association", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("job_id", Integer, ForeignKey("job.id"), index=True),
+    Column("container_type", TEXT),
+    Column("container_name", TEXT),
+    Column("container_info", JSONType, nullable=True),
+    Column("created_time", DateTime, default=now),
+    Column("modified_time", DateTime, default=now, onupdate=now))
+
+def upgrade(migrate_engine):
+    print(__doc__)
+    metadata.bind = migrate_engine
+    metadata.reflect()
+
+    try:
+        realtime_tool.create()
+    except Exception:
+        log.exception("Failed to create realtime_tool table")
+
+    try:
+        job_container_association.create()
+    except Exception:
+        log.exception("Failed to create job_container_association table")
+
+    try:
+        realtime_tool_entry_point.create()
+    except Exception:
+        log.exception("Failed to create realtime_tool_entry_point table")
+
+def downgrade(migrate_engine):
+    metadata.bind = migrate_engine
+    metadata.reflect()
+
+    try:
+        realtime_tool.drop()
+    except Exception:
+        log.exception("Failed to drop realtime_tool table")
+
+    try:
+        job_container_association.drop()
+    except Exception:
+        log.exception("Failed to drop job_container_association table")
+
+    try:
+        realtime_tool_entry_point.drop()
+    except Exception:
+        log.exception("Failed to drop realtime_tool_entry_point table")
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 80f27fcbb69..0e827c46396 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -32,6 +32,7 @@ from galaxy.queue_worker import send_control_task
 from galaxy.tools.actions import DefaultToolAction
 from galaxy.tools.actions.data_manager import DataManagerToolAction
 from galaxy.tools.actions.data_source import DataSourceToolAction
+from galaxy.tools.actions.realtime import RealTimeToolAction
 from galaxy.tools.actions.model_operations import ModelOperationToolAction
 from galaxy.tools.deps import (
     CachedDependencyManager,
@@ -512,7 +513,9 @@ class Tool(Dictifiable):
         """Indicates this tool's runtime requires Galaxy's Python environment."""
         # All special tool types (data source, history import/export, etc...)
         # seem to require Galaxy's Python.
-        if self.tool_type not in ["default", "manage_data"]:
+        # FIXME: the (instantiated) tool class should emit this behavior, and not
+        #        use inspection by string check
+        if self.tool_type not in ["default", "manage_data", "realtime"]:
             return True
 
         if self.tool_type == "manage_data" and self.profile < 18.09:
@@ -763,6 +766,7 @@ class Tool(Dictifiable):
         self.__parse_trackster_conf(tool_source)
         # Record macro paths so we can reload a tool if any of its macro has changes
         self._macro_paths = tool_source.macro_paths()
+        self.tool_ports = tool_source.parse_tool_ports()
 
     def __parse_legacy_features(self, tool_source):
         self.code_namespace = dict()
@@ -1410,7 +1414,8 @@ class Tool(Dictifiable):
                         job_errors=execution_tracker.execution_errors,
                         jobs=execution_tracker.successful_jobs,
                         output_collections=execution_tracker.output_collections,
-                        implicit_collections=execution_tracker.implicit_collections)
+                        implicit_collections=execution_tracker.implicit_collections,
+                        view_result=execution_tracker.view_result)
 
     def handle_single_execution(self, trans, rerun_remap_job_id, execution_slice, history, execution_cache=None, completed_job=None, collection_info=None):
         """
@@ -1839,7 +1844,8 @@ class Tool(Dictifiable):
         tool_dict['panel_section_id'], tool_dict['panel_section_name'] = self.get_panel_section()
 
         tool_class = self.__class__
-        regular_form = tool_class == Tool or isinstance(self, DatabaseOperationTool)
+        # FIXME: the Tool class should declare directly, instead of ad hoc inspection
+        regular_form = tool_class == Tool or isinstance(self, (DatabaseOperationTool, RealTimeTool))
         tool_dict["form_style"] = "regular" if regular_form else "special"
 
         return tool_dict
@@ -1979,6 +1985,10 @@ class Tool(Dictifiable):
             else:
                 group_inputs[input_index] = tool_dict
 
+    def get_view_result(self, job=None, dataset=None,  **kwd):
+        """Returns a special view for a tool and a set of provided objects."""
+        return None
+
     def _get_job_remap(self, job):
         if job:
             if job.state == job.states.ERROR:
@@ -2296,6 +2306,18 @@ class ImportHistoryTool(Tool):
     tool_type = 'import_history'
 
 
+class RealTimeTool(Tool):
+    tool_type = 'realtime'
+    default_tool_action = RealTimeToolAction
+
+    def get_view_result(self, job=None):
+        if job:
+            realtimetool = job.realtime_tool
+            if realtimetool:
+                return url_for(controller='realtime', action='index', realtime_id=self.app.security.encode_id(realtimetool.id))
+        return None
+
+
 class DataManagerTool(OutputParameterJSONTool):
     tool_type = 'manage_data'
     default_tool_action = DataManagerToolAction
@@ -2913,7 +2935,7 @@ class FilterFromFileTool(DatabaseOperationTool):
 
 # Populate tool_type to ToolClass mappings
 tool_types = {}
-for tool_class in [Tool, SetMetadataTool, OutputParameterJSONTool,
+for tool_class in [Tool, SetMetadataTool, OutputParameterJSONTool, RealTimeTool,
                    DataManagerTool, DataSourceTool, AsyncDataSourceTool,
                    UnzipCollectionTool, ZipCollectionTool, MergeCollectionTool, RelabelFromFileTool, FilterFromFileTool,
                    BuildListCollectionTool, ExtractDatasetCollectionTool,
diff --git a/lib/galaxy/tools/actions/realtime.py b/lib/galaxy/tools/actions/realtime.py
new file mode 100644
index 00000000000..0a997f97015
--- /dev/null
+++ b/lib/galaxy/tools/actions/realtime.py
@@ -0,0 +1,15 @@
+import logging
+
+from . import DefaultToolAction
+
+log = logging.getLogger(__name__)
+
+
+class RealTimeToolAction(DefaultToolAction):
+    """Tool action used for RealTime Tools"""
+
+    def execute(self, tool, trans, **kwds):
+        rval = super(RealTimeToolAction, self).execute(tool, trans, **kwds)
+        job, output_dict = rval
+        rtt = trans.app.realtime_manager.create_realtime(trans, job, tool)
+        return rval
diff --git a/lib/galaxy/tools/deps/container_classes.py b/lib/galaxy/tools/deps/container_classes.py
index ecb10f46362..55b678d550e 100644
--- a/lib/galaxy/tools/deps/container_classes.py
+++ b/lib/galaxy/tools/deps/container_classes.py
@@ -5,8 +5,12 @@ from abc import (
     abstractmethod
 )
 
+from uuid import uuid4
+
 import six
 
+from logging import getLogger
+
 from galaxy.containers.docker_model import DockerVolume
 from galaxy.util import (
     asbool,
@@ -21,6 +25,8 @@ from .requirements import (
     DEFAULT_CONTAINER_SHELL,
 )
 
+log = getLogger(__name__)
+
 DOCKER_CONTAINER_TYPE = "docker"
 SINGULARITY_CONTAINER_TYPE = "singularity"
 
@@ -57,13 +63,15 @@ EOF
 @six.add_metaclass(ABCMeta)
 class Container(object):
 
-    def __init__(self, container_id, app_info, tool_info, destination_info, job_info, container_description):
+    def __init__(self, container_id, app_info, tool_info, destination_info, job_info, container_description, container_name=None):
         self.container_id = container_id
         self.app_info = app_info
         self.tool_info = tool_info
         self.destination_info = destination_info
         self.job_info = job_info
         self.container_description = container_description
+        self.container_name = container_name
+        self.container_info = {}
 
     def prop(self, name, default):
         destination_name = "docker_%s" % name
@@ -85,6 +93,10 @@ class Container(object):
         input command in container.
         """
 
+    def set_name(self, job_wrapper=None):
+        self.container_name = uuid4().get_hex()
+        job_wrapper.set_container(self)
+
 
 def preprocess_volumes(volumes_raw_str, container_type):
     """Process Galaxy volume specification string to either Docker or Singularity specification.
@@ -251,8 +263,13 @@ class DockerContainer(Container, HasDockerLikeVolumes):
             auto_rm=asbool(self.prop("auto_rm", docker_util.DEFAULT_AUTO_REMOVE)),
             set_user=self.prop("set_user", docker_util.DEFAULT_SET_USER),
             run_extra_arguments=self.prop("run_extra_arguments", docker_util.DEFAULT_RUN_EXTRA_ARGUMENTS),
+            guest_ports=self.tool_info.guest_ports,
+            container_name=self.container_name,
             **docker_host_props
         )
+        self.container_info['commands'] = {}
+        for cmd in ['inspect', 'port', 'kill', 'stop']:
+            self.container_info['commands'][cmd]=docker_util.build_docker_simple_command(cmd, container_name=self.container_name, **docker_host_props)
         return "%s\n%s" % (cache_command, run_command)
 
     def __cache_from_file_command(self, cached_image_file, docker_host_props):
@@ -333,6 +350,8 @@ class SingularityContainer(Container, HasDockerLikeVolumes):
             env=env,
             working_directory=working_directory,
             run_extra_arguments=self.prop("run_extra_arguments", singularity_util.DEFAULT_RUN_EXTRA_ARGUMENTS),
+            guest_ports=self.tool_info.guest_ports,
+            container_name=self.container_name,
             **self.get_singularity_target_kwds()
         )
         return run_command
diff --git a/lib/galaxy/tools/deps/dependencies.py b/lib/galaxy/tools/deps/dependencies.py
index bc1820cf52c..6a66a555a46 100644
--- a/lib/galaxy/tools/deps/dependencies.py
+++ b/lib/galaxy/tools/deps/dependencies.py
@@ -36,7 +36,7 @@ class ToolInfo(object):
     # variables they can consume (e.g. JVM options, license keys, etc..)
     # and add these to env_path_through
 
-    def __init__(self, container_descriptions=None, requirements=None, requires_galaxy_python_environment=False, env_pass_through=["GALAXY_SLOTS"]):
+    def __init__(self, container_descriptions=None, requirements=None, requires_galaxy_python_environment=False, env_pass_through=["GALAXY_SLOTS"], guest_ports=None):
         if container_descriptions is None:
             container_descriptions = []
         if requirements is None:
@@ -45,6 +45,7 @@ class ToolInfo(object):
         self.requirements = requirements
         self.requires_galaxy_python_environment = requires_galaxy_python_environment
         self.env_pass_through = env_pass_through
+        self.guest_ports = guest_ports
 
 
 class JobInfo(object):
diff --git a/lib/galaxy/tools/deps/docker_util.py b/lib/galaxy/tools/deps/docker_util.py
index d88c9b338e2..f4353a8b44c 100644
--- a/lib/galaxy/tools/deps/docker_util.py
+++ b/lib/galaxy/tools/deps/docker_util.py
@@ -81,6 +81,23 @@ def build_docker_images_command(truncate=True, **kwds):
 def build_docker_load_command(**kwds):
     return command_shell("load", [])
 
+def build_docker_simple_command(
+    command,
+    docker_cmd=DEFAULT_DOCKER_COMMAND,
+    sudo=DEFAULT_SUDO,
+    sudo_cmd=DEFAULT_SUDO_COMMAND,
+    container_name=None,
+    **kwd
+):
+    command_parts = _docker_prefix(
+        docker_cmd=docker_cmd,
+        sudo=sudo,
+        sudo_cmd=sudo_cmd,
+    )
+    command_parts.append(command)
+    command_parts.append(container_name or '{CONTAINER_NAME}')
+    return " ".join(command_parts)
+
 
 def build_docker_run_command(
     container_command,
@@ -102,6 +119,8 @@ def build_docker_run_command(
     auto_rm=DEFAULT_AUTO_REMOVE,
     set_user=DEFAULT_SET_USER,
     host=DEFAULT_HOST,
+    guest_ports=False,
+    container_name=None
 ):
     command_parts = _docker_prefix(
         docker_cmd=docker_cmd,
@@ -118,6 +137,16 @@ def build_docker_run_command(
         # e.g. -e "GALAXY_SLOTS=$GALAXY_SLOTS"
         # These are environment variable expansions so we don't quote these.
         command_parts.extend(["-e", env_directive])
+    if guest_ports is True:
+        # When is True, expose all ports
+        command_parts.append("-P")
+    elif guest_ports:
+        if not isinstance(guest_ports, list):
+            guest_ports = [guest_ports]
+        for guest_port in guest_ports:
+            command_parts.extend(["-p", guest_port])
+    if container_name:
+        command_parts.extend(["--name", container_name])
     for volume in volumes:
         # These are environment variable expansions so we don't quote these.
         volume_str = str(volume)
diff --git a/lib/galaxy/tools/deps/singularity_util.py b/lib/galaxy/tools/deps/singularity_util.py
index ba281f8880f..a36c1e81ede 100644
--- a/lib/galaxy/tools/deps/singularity_util.py
+++ b/lib/galaxy/tools/deps/singularity_util.py
@@ -40,6 +40,8 @@ def build_singularity_run_command(
     run_extra_arguments=DEFAULT_RUN_EXTRA_ARGUMENTS,
     sudo=DEFAULT_SUDO,
     sudo_cmd=DEFAULT_SUDO_COMMAND,
+    guest_ports=False,
+    container_name=None
 ):
     command_parts = []
     # http://singularity.lbl.gov/docs-environment-metadata
diff --git a/lib/galaxy/tools/execute.py b/lib/galaxy/tools/execute.py
index d2e722cdae0..f513e5ac8ef 100644
--- a/lib/galaxy/tools/execute.py
+++ b/lib/galaxy/tools/execute.py
@@ -356,6 +356,15 @@ class ExecutionTracker(object):
             job_assoc.job_id = job.id
             self.trans.sa_session.add(job_assoc)
 
+    @property
+    def view_result(self):
+        rval = []
+        for job in self.successful_jobs:
+            result = self.tool.get_view_result(job=job)
+            if result:
+                rval.append(result)
+        return rval
+
 
 # Seperate these because workflows need to track their jobs belong to the invocation
 # in the database immediately and they can be recovered.
diff --git a/lib/galaxy/tools/parser/interface.py b/lib/galaxy/tools/parser/interface.py
index 2cd31a1aa6f..35065dcdd8a 100644
--- a/lib/galaxy/tools/parser/interface.py
+++ b/lib/galaxy/tools/parser/interface.py
@@ -117,6 +117,11 @@ class ToolSource(object):
         adjacent to the tool).
         """
 
+    @abstractmethod
+    def parse_tool_ports(self):
+        """ Return port templates to expose.
+        """
+
     def parse_redirect_url_params_elem(self):
         """ Return an XML element describing redirect_url_params.
 
diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py
index 997eb760406..b3083eaf73e 100644
--- a/lib/galaxy/tools/parser/xml.py
+++ b/lib/galaxy/tools/parser/xml.py
@@ -187,6 +187,20 @@ class XmlToolSource(ToolSource):
             hidden = string_as_bool(hidden)
         return hidden
 
+    def parse_tool_ports(self):
+        # TODO: allow ports to be templated, for now, we will just use static value
+        tool_ports_el = self.root.find("tool_ports")
+        tool_ports = []
+        if tool_ports_el is None:
+            return tool_ports
+        for tool_port_el in tool_ports_el.findall("port"):
+            port = tool_port_el.text.strip()
+            entry_path = tool_port_el.find("url")
+            if entry_path is not None:
+                entry_path=entry_path.text.strip()
+            tool_ports.append(dict(port=port, url=entry_path, name=tool_port_el.get('name', None)))
+        return tool_ports
+
     def parse_redirect_url_params_elem(self):
         return self.root.find("redirect_url_params")
 
@@ -201,7 +215,6 @@ class XmlToolSource(ToolSource):
         for option_elem in root.findall("options"):
             if key in option_elem.attrib:
                 return string_as_bool(option_elem.get(key))
-
         return default
 
     @property
diff --git a/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py b/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
new file mode 100644
index 00000000000..2c070fa8085
--- /dev/null
+++ b/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
@@ -0,0 +1,31 @@
+import uwsgi
+import sqlite3
+
+
+realtime_db_file = uwsgi.opt[ "sessions" ]
+db_conn = sqlite3.connect(realtime_db_file)
+
+
+def key_type_token_mapper(key, key_type, token, route_extra, url):
+    global db_conn
+    #print 'key %s key_type %s token %s route_extra %s url %s\n' % (key, key_type, token, route_extra, url)
+    if key and key_type and token:
+        #qlite3.ProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id x and this is thread id y.
+        # So try upto 2 times
+        for i in range(2):
+            # Order by rowid gives us the last row added
+            print "row %s" % (i)
+            try:
+                row = db_conn.execute( "select host, port from gxrtproxy where key=? and key_type=? and token=? order by rowid desc limit 1", ( key,key_type,token) ).fetchone()
+                if row:
+                    rval = '%s:%s' % (tuple(row))
+                    return rval.encode()
+               
+                break
+            except sqlite3.ProgrammingError as e:
+                db_conn = sqlite3.connect(realtime_db_file)
+                continue
+            break
+    return None
+
+uwsgi.register_rpc('rtt_key_type_token_mapper', key_type_token_mapper)
diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py
index a7319f4898c..de6ea3db900 100644
--- a/lib/galaxy/webapps/galaxy/api/tools.py
+++ b/lib/galaxy/webapps/galaxy/api/tools.py
@@ -553,6 +553,7 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
             output_dict['output_name'] = output_name
             rval['implicit_collections'].append(output_dict)
 
+        rval['view_result'] = vars.get('view_result', None)
         return rval
 
     #
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
new file mode 100644
index 00000000000..8c35553620a
--- /dev/null
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -0,0 +1,148 @@
+"""
+Provides web interaction with RealTimeTools
+"""
+import uwsgi
+import logging
+from collections import OrderedDict
+from datetime import datetime, timedelta
+
+from galaxy import (
+    model,
+    util,
+    web
+)
+from galaxy.web import url_for
+from galaxy.web.base.controller import (
+    BaseUIController,
+)
+
+from galaxy.web.framework.helpers import (
+    grids,
+    time_ago,
+)
+
+log = logging.getLogger(__name__)
+
+
+class RealTimeToolEntryPointListGrid(grids.Grid):
+    def get_url_args(item):
+        """
+        Returns dictionary used to create item link.
+        """
+        url_kwargs = dict(controller="realtime", action="access_entry_point_ri", entry_point_id=item.id)
+        return url_kwargs
+
+    use_panels = True
+    title = "Available RealTimeTools"
+    model_class = model.RealTimeToolEntryPoint
+    default_filter = {"name": "All"}
+    default_sort_key = "-update_time"
+    columns = [
+        grids.TextColumn("Name", key="name", filterable="advanced", link=get_url_args),
+        grids.GridColumn("Active", key="active"),
+        grids.GridColumn("Created", key="created_time", format=time_ago),
+        grids.GridColumn("Last Updated", key="modified_time", format=time_ago),
+    ]
+    columns.append(
+        grids.MulticolFilterColumn(
+            "Search",
+            cols_to_filter=[columns[0] ],
+            key="free-text-search", visible=False, filterable="standard"
+        )
+    )
+    operations = [
+        grids.GridOperation("Open", condition=(lambda item: item.active), async_compatible=False),
+        grids.GridOperation("Stop", condition=(lambda item: item.active), async_compatible=False),
+    ]
+
+    def build_initial_query(self, trans, **kwargs):
+        # Get list of user's active RealTimeTools
+        return trans.app.realtime_manager.get_nonterminal_for_user_by_trans(trans)
+
+
+
+class RealTime(BaseUIController):
+    _rtt_ep_grid = RealTimeToolEntryPointListGrid()
+
+    def __get_realtime_view_host(trans, realtime_entry):
+        if not isinstance(realtime_entry, trans.app.model.RealTimeToolEntryPoint):
+            realtime_entry = trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).get(trans.security.decode_id(realtime_entry_id))
+        return "%s:%s" % (realtime_entry.host, realtime_entry.port)
+
+
+    @web.expose
+    def index(self, trans, realtime_id=None, **kwd):
+        realtime = None
+        if realtime_id:
+            realtime_id = self.decode_id(realtime_id)
+            realtime = trans.sa_session.query(trans.app.model.RealTimeTool).get(realtime_id)
+        if realtime:
+            if trans.app.realtime_manager.can_access_realtime(trans, realtime):
+                if realtime.entry_points:
+                    if len(realtime.entry_points)==1:
+                        return self.access_entry_point(trans, entry_point_id=trans.security.encode_id(realtime.entry_points[0].id))
+                    else:
+                        return trans.response.send_redirect(url_for(controller="realtime", action="list"))
+            else:
+                return trans.show_error_message('Access not authorized.')
+        return trans.show_error_message('RealTimeTool instance not found')
+
+    @web.expose
+    def access_entry_point_ri(self, trans, entry_point_id=None):
+        return trans.response.send_redirect(url_for(controller='realtime', action='access_entry_point', entry_point_id=trans.security.encode_id(entry_point_id)))
+
+    @web.expose
+    def access_entry_point(self, trans, entry_point_id=None):
+        if entry_point_id:
+            entry_point_id = self.decode_id(entry_point_id)
+            entry_point = trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).get(entry_point_id)
+            if trans.app.realtime_manager.can_access_entry_point(trans, entry_point):
+                if entry_point.realtime.active and entry_point.configured:
+                    rval = (url_for( '%s/%s/%s/%s/' % (trans.app.config.realtime_prefix, entry_point.__class__.__name__, trans.security.encode_id(entry_point.id), entry_point.token)))
+                    if entry_point.entry_url:
+                        rval = '%s%s' % (rval, entry_point.entry_url)
+                    return trans.response.send_redirect(rval)
+                else:
+                    return trans.show_error_message('RealTimeTool is not active. If you recently launched this tool it may not be ready yet, please wait a moment and refresh this page.')
+        return trans.show_error_message('Access not authorized.')
+
+    def _get_job_for_dataset(self, trans, dataset_id):
+        '''
+        Return the job for the given dataset. This will throw an error if the
+        dataset is either nonexistent or inaccessible to the user.
+        '''
+        hda = trans.sa_session.query(trans.app.model.HistoryDatasetAssociation).get(self.decode_id(dataset_id))
+        return hda.creating_job
+
+    def _get_job(self, trans, job_id):
+        '''
+        Return the job for the given dataset. This will throw an error if the
+        dataset is either nonexistent or inaccessible to the user.
+        '''
+        return trans.sa_session.query(trans.app.model.Job).get(self.decode_id(job_id))
+
+    @web.expose_api
+    def list(self, trans, **kwargs):
+        """List all available realtimetools"""
+        return self._rtt_ep_grid(trans)
+        rtts = trans.app.realtime_manager.get_nonterminal_for_user_by_trans(trans)
+        eps = []
+        if rtts:
+            for rtt in rtts:
+                for ep in rtt.entry_points:
+                    eps.append(self.encode_all_ids(trans, ep.to_dict(), True))
+        return self._rtt_ep_grid(eps)
+
+    @web.expose_api
+    def stop(self, trans, realtime_id=None, realtime_entry_id=None):
+        """List all available realtimetools"""
+        if realtime_id:
+            rtt = trans.app.model.RealTimeTool.get(trans.security.decode_id(realtime_id))
+        elif realtime_entry_id:
+            rtt = trans.app.model.RealTimeToolEntryPoint.get(trans.security.decode_id(realtime_entry_id)).realtime
+        else:
+            return dict(message="Can't stop: RealTimeTool not provided.",state='error')
+        stopped = trans.app.realtime_manager.stop(realtime)
+        if stopped:
+            return dict(message="RealTimeTool has been stopped",state='ok')
+        return dict(message="Unable to stop RealTimeTool",state='error')
diff --git a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py
index 5a456179d4c..a6b791b6b57 100644
--- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py
+++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py
@@ -61,7 +61,8 @@ class ToolRunner(BaseUIController):
                                                         redirect=redirect))
         if not tool.allow_user_access(trans.user):
             return __tool_404__()
-        if tool.tool_type == 'default':
+        # FIXME: Tool class should define behavior
+        if tool.tool_type in ['default', 'realtime']:
             return trans.response.send_redirect(url_for(controller='root', tool_id=tool_id))
 
         # execute tool without displaying form (used for datasource tools)

From edeca763f966064c671a05bfbd16f2463110f11c Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 8 Mar 2019 13:49:23 -0500
Subject: [PATCH 002/114] Flake8

---
 lib/galaxy/datatypes/realtime.py              |  3 +-
 lib/galaxy/jobs/__init__.py                   |  2 +-
 lib/galaxy/jobs/runners/__init__.py           |  6 +--
 lib/galaxy/jobs/runners/local.py              | 16 ++++----
 lib/galaxy/managers/realtime.py               | 41 ++++++++-----------
 lib/galaxy/model/__init__.py                  |  8 ++--
 .../migrate/versions/0147_realtime_tool.py    |  4 +-
 lib/galaxy/tools/__init__.py                  |  2 +-
 lib/galaxy/tools/actions/realtime.py          |  2 +-
 lib/galaxy/tools/deps/container_classes.py    |  2 +-
 lib/galaxy/tools/deps/docker_util.py          |  1 +
 lib/galaxy/tools/parser/xml.py                |  2 +-
 .../realtime/rpc/key_type_token_mapping.py    | 13 +++---
 .../webapps/galaxy/controllers/realtime.py    | 27 ++++--------
 14 files changed, 57 insertions(+), 72 deletions(-)

diff --git a/lib/galaxy/datatypes/realtime.py b/lib/galaxy/datatypes/realtime.py
index b9ceb5e1bd8..63df3956c35 100644
--- a/lib/galaxy/datatypes/realtime.py
+++ b/lib/galaxy/datatypes/realtime.py
@@ -2,6 +2,8 @@
 from __future__ import print_function
 
 import logging
+import os
+import sys
 
 from text import Html
 from galaxy.web import url_for
@@ -19,7 +21,6 @@ class Realtime(Html):
         dataset.blurb = "RealtimeTool"
         dataset.peek = 'Finished RealTimeTool'
 
-
     def display_peek(self, dataset):
         try:
             if not dataset.peek:
diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py
index d4f9f588a2b..c26f332e7ff 100644
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -1893,7 +1893,7 @@ class JobWrapper(HasResourceParameters):
 
     def set_container(self, container):
         if container:
-            cont = model.JobContainerAssociation( job=self.get_job(), container_type=container.container_type, container_name=container.container_name, container_info=container.container_info)
+            cont = model.JobContainerAssociation(job=self.get_job(), container_type=container.container_type, container_name=container.container_name, container_info=container.container_info)
             self.sa_session.add(cont)
             self.sa_session.flush()
 
diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py
index 349fb78f5ad..ed324d040ef 100644
--- a/lib/galaxy/jobs/runners/__init__.py
+++ b/lib/galaxy/jobs/runners/__init__.py
@@ -260,7 +260,7 @@ class BaseJobRunner(object):
         container = self._find_container(job_wrapper)
         if not container and job_wrapper.requires_containerization:
             raise Exception("Failed to find a container when required, contact Galaxy admin.")
-        built=build_command(
+        built = build_command(
             self,
             job_wrapper,
             include_metadata=include_metadata,
@@ -273,7 +273,7 @@ class BaseJobRunner(object):
         stored_container = job_wrapper.get_job().container
         if container and stored_container:
             if container.container_info:
-                stored_container.container_info=container.container_info
+                stored_container.container_info = container.container_info
         return built
 
     def get_work_dir_outputs(self, job_wrapper, job_working_directory=None, tool_working_directory=None):
@@ -429,7 +429,7 @@ class BaseJobRunner(object):
         )
 
         destination_info = job_wrapper.job_destination.params
-        container=self.app.container_finder.find_container(
+        container = self.app.container_finder.find_container(
             tool_info,
             destination_info,
             job_info
diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py
index 2337a0aeae8..144c851c8f5 100644
--- a/lib/galaxy/jobs/runners/local.py
+++ b/lib/galaxy/jobs/runners/local.py
@@ -269,7 +269,7 @@ class LocalJobRunner(BaseJobRunner):
             if cont:
                 if cont.container_type == 'docker':
                     # Do we need to do inspect, or is only ports ok?
-                    inspect=None
+                    inspect = None
                     for i in range(1, 11):
                         with tempfile.TemporaryFile() as stdout_file:
                             exit_code = subprocess.call(cont.container_info['commands']['inspect'],
@@ -281,33 +281,32 @@ class LocalJobRunner(BaseJobRunner):
                                 stdout_file.seek(0)
                                 inspect = stdout_file.read()
                                 break
-                        t=2*i
+                        t = 2 * i
                         log.debug("Container not found, sleeping for %s seconds.", t)
                         sleep(t)
                     cont.container_info['inspect'] = json.loads(inspect)
                     rtt = job.realtime_tool
                     if rtt:
-                        ports_raw=None
+                        ports_raw = None
                         for i in range(1, 11):
                             with tempfile.TemporaryFile() as stdout_file:
                                 exit_code = subprocess.call(cont.container_info['commands']['port'],
                                                         shell=True,
                                                         stdout=stdout_file,
-                                                        #stderr=stderr_file,
                                                         env=self._environ,
                                                         preexec_fn=os.setpgrp)
                                 if exit_code == 0:
                                     stdout_file.seek(0)
                                     ports_raw = stdout_file.read()
                                     break
-                            t=2*i
+                            t = 2 * i
                             log.debug("Container not found, sleeping for %s seconds.", t)
                             sleep(t)
 
                         for line in ports_raw.strip().split('\n'):
-                            #TODO: configure all ports at once, instead of individually
-                            #A port could e.g. be mapped multiple times, with diff entry URL
-                            tool, host =  line.split(" -> ", 1)
+                            # TODO: configure all ports at once, instead of individually
+                            # A port could e.g. be mapped multiple times, with diff entry URL
+                            tool, host = line.split(" -> ", 1)
                             hostname, port = host.split(':')
                             tool_p, tool_prot = tool.split("/")
                             ep = self.app.realtime_manager.configure_entry_point(rtt, tool_port=tool_p, host=hostname, port=port, protocol=tool_prot)
@@ -317,7 +316,6 @@ class LocalJobRunner(BaseJobRunner):
                                 job_wrapper.sa_session.add(ep)
                                 job_wrapper.sa_session.flush()
 
-
     def __poll_if_needed(self, proc, job_wrapper, job_id):
         # Only poll if needed (i.e. job limits are set)
         self.__handle_container(job_wrapper)
diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index 317e88d37ab..f7d3a206b0e 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -1,5 +1,4 @@
 import logging
-import json
 import sqlite3
 
 
@@ -7,8 +6,6 @@ from six import string_types
 from sqlalchemy import or_
 
 
-from galaxy import model
-
 from galaxy.util.filelock import FileLock
 
 
@@ -32,7 +29,7 @@ class RealtimeSqlite(object):
                 select = '''SELECT token, host, port, info
                             FROM %s
                             WHERE key=? and key_type=?''' % (DATABASE_TABLE_NAME)
-                c.execute(select, (key,key_type,))
+                c.execute(select, (key, key_type,))
                 try:
                     token, host, port, info = c.fetchone()
                 except TypeError:
@@ -48,7 +45,7 @@ class RealtimeSqlite(object):
             finally:
                 conn.close()
 
-    def save(self, key, key_type, token, host, port, info=None):#, route_name):
+    def save(self, key, key_type, token, host, port, info=None):
         """
         Writeout a key, key_type, token, value store that is can be used for coordinating
         with external resources.
@@ -56,7 +53,6 @@ class RealtimeSqlite(object):
         assert key, ValueError("A non-zero length key is required.")
         assert key_type, ValueError("A non-zero length key_type is required.")
         assert token, ValueError("A non-zero length token is required.")
-        rval = None
         with FileLock(self.sqlite_filename):
             conn = sqlite3.connect(self.sqlite_filename)
             try:
@@ -71,11 +67,11 @@ class RealtimeSqlite(object):
                                   port integer,
                                   info text,
                                   PRIMARY KEY (key, key_type)
-                                  )'''%(DATABASE_TABLE_NAME))
+                                  )''' % (DATABASE_TABLE_NAME))
                 except Exception:
                     pass
                 delete = '''DELETE FROM %s WHERE key=? and key_type=?''' % (DATABASE_TABLE_NAME)
-                c.execute(delete, (key,key_type,))
+                c.execute(delete, (key, key_type,))
                 insert = '''INSERT INTO %s
                             (key, key_type, token, host, port, info)
                             VALUES (?, ?, ?, ?, ?, ?)''' % (DATABASE_TABLE_NAME)
@@ -97,8 +93,6 @@ class RealtimeSqlite(object):
         return self.save(self.encode_id(entry_point.id), entry_point.__class__.__name__, entry_point.token, entry_point.host, entry_point.port, None)
 
 
-
-
 class RealTimeManager(object):
     """
     Manager for dealing with RealTimeTools
@@ -107,16 +101,17 @@ class RealTimeManager(object):
     def __init__(self, app):
         self.model = app.model
         self.security = app.security
-        self.propagator = RealtimeSqlite(app.config.proxy_session_map, app.security.encode_id) # Use existing db, but new table
+        self.propagator = RealtimeSqlite(app.config.proxy_session_map, app.security.encode_id)
 
-    #FIXME: Do we really want to directly associate a single dataset?
+    # FIXME: Do we really want to directly associate a single dataset?
     # Probably Not
     def guess_associated_dataset(self, datasets):
         for dataset in datasets:
-            dataset = dataset.dataset #AttributeError: 'JobToOutputDatasetAssociation' object has no attribute 'ext'
+            dataset = dataset.dataset
             if dataset.ext == REALTIMETOOL_DATASET_EXT:
                 return dataset
         return None
+
     def create_entry_points(self, trans, rtt, tool, flush=True):
         for entry in tool.tool_ports:
             rtt_entry = self.model.RealTimeToolEntryPoint(realtime=rtt, tool_port=entry['port'], entry_url=entry['url'], name=entry['name'])
@@ -126,33 +121,30 @@ class RealTimeManager(object):
 
     def configure_entry_point(self, rtt, tool_port=None, host=None, port=None, protocol=None):
         if tool_port is not None:
-            tool_port=int(tool_port)
+            tool_port = int(tool_port)
         not_configured = []
         for entry in rtt.entry_points:
             if entry.tool_port == tool_port:
-                entry.host=host
-                entry.port=port
-                entry.protocol=protocol
+                entry.host = host
+                entry.port = port
+                entry.protocol = protocol
                 entry.configured = True
                 self.save_entry_point(entry)
                 return entry
             elif not entry.configured:
                 not_configured.append(entry)
-
         # TODO: port not found in entry, add to random one?
         log.warning('tool port not found! %s', str(not_configured))
-        return None 
-        
+        return None
 
     def save_entry_point(self, entry_point):
         """
         Writeout a key, key_type, token, value store that is used validating access
         """
         self.propagator.save_entry_point(entry_point)
-   
+
     def create_realtime(self, trans, job, tool):
         # create from initial job
-        datasets = None
         if job:
             realtime_tool = self.model.RealTimeTool(job=job, user=trans.user, session=trans.galaxy_session, dataset=self.guess_associated_dataset(job.get_output_datasets()))
             trans.sa_session.add(realtime_tool)
@@ -160,13 +152,14 @@ class RealTimeManager(object):
                 self.create_entry_points(trans, realtime_tool, tool, flush=False)
             trans.sa_session.flush()
         else:
-            log.warning( 'Called RealTimeManager.create_realtime, but job is None' )
+            log.warning('Called RealTimeManager.create_realtime, but job is None')
 
     def get_nonterminal_for_user_by_trans(self, trans):
         if trans.user:
             jobs = trans.sa_session.query(trans.app.model.Job).filter(trans.app.model.Job.user == trans.user)
         else:
             jobs = trans.sa_session.query(trans.app.model.Job).filter(trans.app.model.Job.session_id == trans.get_galaxy_session().id)
+
         def build_and_apply_filters(query, objects, filter_func):
             if objects is not None:
                 if isinstance(objects, string_types):
@@ -179,7 +172,7 @@ class RealTimeManager(object):
             return query
         jobs = build_and_apply_filters(jobs, trans.app.model.Job.non_ready_states, lambda s: trans.app.model.Job.state == s)
         rtts = trans.sa_session.query(trans.app.model.RealTimeTool).filter(trans.app.model.RealTimeTool.job_id.in_([job.id for job in jobs]))
-        return trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).filter(trans.app.model.RealTimeToolEntryPoint.realtime_id.in_([ rtt.id for rtt in rtts ]))
+        return trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).filter(trans.app.model.RealTimeToolEntryPoint.realtime_id.in_([rtt.id for rtt in rtts]))
 
     def can_access_realtime(self, trans, realtime):
         if realtime:
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 33873e75548..009f50b0f3e 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -1313,12 +1313,14 @@ class RealTimeTool(RepresentById):
         self.user = user
         self.galaxy_session = session
         self.dataset = dataset
+
     @property
     def active(self):
         # FIXME: don't included queued?
         return not self.job.finished
+
     def to_dict(self, *args, **kwds):
-        return dict(job_id=self.job.id, user_id=self.user.id,galaxy_session_id=self.galaxy_session.id,dataset_id=self.dataset.id,active=self.active)#=self.realtime.to_dict(*args, **kwds))
+        return dict(job_id=self.job.id, user_id=self.user.id, galaxy_session_id=self.galaxy_session.id, dataset_id=self.dataset.id, active=self.active)
 
 
 class RealTimeToolEntryPoint(RepresentById):
@@ -1338,8 +1340,8 @@ class RealTimeToolEntryPoint(RepresentById):
 
     def to_dict(self, *args, **kwds):
         rval = dict(realtime_id=self.realtime.id)
-        for val in ['id', 'name','token','tool_port','entry_url','host','port','protocol','configured', 'created_time', 'modified_time', 'active']:
-            rval[val]=getattr(self, val)
+        for val in ['id', 'name', 'token', 'tool_port', 'entry_url', 'host', 'port', 'protocol', 'configured', 'created_time', 'modified_time', 'active']:
+            rval[val] = getattr(self, val)
         return rval
 
     @property
diff --git a/lib/galaxy/model/migrate/versions/0147_realtime_tool.py b/lib/galaxy/model/migrate/versions/0147_realtime_tool.py
index 15b2f535820..5e2764b9a27 100644
--- a/lib/galaxy/model/migrate/versions/0147_realtime_tool.py
+++ b/lib/galaxy/model/migrate/versions/0147_realtime_tool.py
@@ -5,7 +5,7 @@ from __future__ import print_function
 
 import logging
 
-from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, String, Table, TEXT
+from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT
 
 from galaxy.model.custom_types import JSONType
 from galaxy.model.orm.now import now
@@ -49,6 +49,7 @@ job_container_association = Table(
     Column("created_time", DateTime, default=now),
     Column("modified_time", DateTime, default=now, onupdate=now))
 
+
 def upgrade(migrate_engine):
     print(__doc__)
     metadata.bind = migrate_engine
@@ -69,6 +70,7 @@ def upgrade(migrate_engine):
     except Exception:
         log.exception("Failed to create realtime_tool_entry_point table")
 
+
 def downgrade(migrate_engine):
     metadata.bind = migrate_engine
     metadata.reflect()
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 0e827c46396..6fc6a4d415c 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -1985,7 +1985,7 @@ class Tool(Dictifiable):
             else:
                 group_inputs[input_index] = tool_dict
 
-    def get_view_result(self, job=None, dataset=None,  **kwd):
+    def get_view_result(self, job=None, dataset=None, **kwd):
         """Returns a special view for a tool and a set of provided objects."""
         return None
 
diff --git a/lib/galaxy/tools/actions/realtime.py b/lib/galaxy/tools/actions/realtime.py
index 0a997f97015..01af6dd3de6 100644
--- a/lib/galaxy/tools/actions/realtime.py
+++ b/lib/galaxy/tools/actions/realtime.py
@@ -11,5 +11,5 @@ class RealTimeToolAction(DefaultToolAction):
     def execute(self, tool, trans, **kwds):
         rval = super(RealTimeToolAction, self).execute(tool, trans, **kwds)
         job, output_dict = rval
-        rtt = trans.app.realtime_manager.create_realtime(trans, job, tool)
+        trans.app.realtime_manager.create_realtime(trans, job, tool)
         return rval
diff --git a/lib/galaxy/tools/deps/container_classes.py b/lib/galaxy/tools/deps/container_classes.py
index 55b678d550e..c442f95eb34 100644
--- a/lib/galaxy/tools/deps/container_classes.py
+++ b/lib/galaxy/tools/deps/container_classes.py
@@ -269,7 +269,7 @@ class DockerContainer(Container, HasDockerLikeVolumes):
         )
         self.container_info['commands'] = {}
         for cmd in ['inspect', 'port', 'kill', 'stop']:
-            self.container_info['commands'][cmd]=docker_util.build_docker_simple_command(cmd, container_name=self.container_name, **docker_host_props)
+            self.container_info['commands'][cmd] = docker_util.build_docker_simple_command(cmd, container_name=self.container_name, **docker_host_props)
         return "%s\n%s" % (cache_command, run_command)
 
     def __cache_from_file_command(self, cached_image_file, docker_host_props):
diff --git a/lib/galaxy/tools/deps/docker_util.py b/lib/galaxy/tools/deps/docker_util.py
index f4353a8b44c..162eb3a7584 100644
--- a/lib/galaxy/tools/deps/docker_util.py
+++ b/lib/galaxy/tools/deps/docker_util.py
@@ -81,6 +81,7 @@ def build_docker_images_command(truncate=True, **kwds):
 def build_docker_load_command(**kwds):
     return command_shell("load", [])
 
+
 def build_docker_simple_command(
     command,
     docker_cmd=DEFAULT_DOCKER_COMMAND,
diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py
index b3083eaf73e..9602be8442e 100644
--- a/lib/galaxy/tools/parser/xml.py
+++ b/lib/galaxy/tools/parser/xml.py
@@ -197,7 +197,7 @@ class XmlToolSource(ToolSource):
             port = tool_port_el.text.strip()
             entry_path = tool_port_el.find("url")
             if entry_path is not None:
-                entry_path=entry_path.text.strip()
+                entry_path = entry_path.text.strip()
             tool_ports.append(dict(port=port, url=entry_path, name=tool_port_el.get('name', None)))
         return tool_ports
 
diff --git a/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py b/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
index 2c070fa8085..0b9217a3e2c 100644
--- a/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
+++ b/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
@@ -2,30 +2,29 @@ import uwsgi
 import sqlite3
 
 
-realtime_db_file = uwsgi.opt[ "sessions" ]
+realtime_db_file = uwsgi.opt["sessions"]
 db_conn = sqlite3.connect(realtime_db_file)
 
 
 def key_type_token_mapper(key, key_type, token, route_extra, url):
     global db_conn
-    #print 'key %s key_type %s token %s route_extra %s url %s\n' % (key, key_type, token, route_extra, url)
+    # print 'key %s key_type %s token %s route_extra %s url %s\n' % (key, key_type, token, route_extra, url)
     if key and key_type and token:
-        #qlite3.ProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id x and this is thread id y.
+        # sqlite3.ProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id x and this is thread id y.
         # So try upto 2 times
         for i in range(2):
             # Order by rowid gives us the last row added
-            print "row %s" % (i)
             try:
-                row = db_conn.execute( "select host, port from gxrtproxy where key=? and key_type=? and token=? order by rowid desc limit 1", ( key,key_type,token) ).fetchone()
+                row = db_conn.execute("select host, port from gxrtproxy where key=? and key_type=? and token=? order by rowid desc limit 1", (key, key_type, token)).fetchone()
                 if row:
                     rval = '%s:%s' % (tuple(row))
                     return rval.encode()
-               
                 break
-            except sqlite3.ProgrammingError as e:
+            except sqlite3.ProgrammingError:
                 db_conn = sqlite3.connect(realtime_db_file)
                 continue
             break
     return None
 
+
 uwsgi.register_rpc('rtt_key_type_token_mapper', key_type_token_mapper)
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index 8c35553620a..1b843449d8a 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -1,14 +1,10 @@
 """
 Provides web interaction with RealTimeTools
 """
-import uwsgi
 import logging
-from collections import OrderedDict
-from datetime import datetime, timedelta
 
 from galaxy import (
     model,
-    util,
     web
 )
 from galaxy.web import url_for
@@ -46,7 +42,7 @@ class RealTimeToolEntryPointListGrid(grids.Grid):
     columns.append(
         grids.MulticolFilterColumn(
             "Search",
-            cols_to_filter=[columns[0] ],
+            cols_to_filter=[columns[0]],
             key="free-text-search", visible=False, filterable="standard"
         )
     )
@@ -60,16 +56,9 @@ class RealTimeToolEntryPointListGrid(grids.Grid):
         return trans.app.realtime_manager.get_nonterminal_for_user_by_trans(trans)
 
 
-
 class RealTime(BaseUIController):
     _rtt_ep_grid = RealTimeToolEntryPointListGrid()
 
-    def __get_realtime_view_host(trans, realtime_entry):
-        if not isinstance(realtime_entry, trans.app.model.RealTimeToolEntryPoint):
-            realtime_entry = trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).get(trans.security.decode_id(realtime_entry_id))
-        return "%s:%s" % (realtime_entry.host, realtime_entry.port)
-
-
     @web.expose
     def index(self, trans, realtime_id=None, **kwd):
         realtime = None
@@ -79,7 +68,7 @@ class RealTime(BaseUIController):
         if realtime:
             if trans.app.realtime_manager.can_access_realtime(trans, realtime):
                 if realtime.entry_points:
-                    if len(realtime.entry_points)==1:
+                    if len(realtime.entry_points) == 1:
                         return self.access_entry_point(trans, entry_point_id=trans.security.encode_id(realtime.entry_points[0].id))
                     else:
                         return trans.response.send_redirect(url_for(controller="realtime", action="list"))
@@ -98,7 +87,7 @@ class RealTime(BaseUIController):
             entry_point = trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).get(entry_point_id)
             if trans.app.realtime_manager.can_access_entry_point(trans, entry_point):
                 if entry_point.realtime.active and entry_point.configured:
-                    rval = (url_for( '%s/%s/%s/%s/' % (trans.app.config.realtime_prefix, entry_point.__class__.__name__, trans.security.encode_id(entry_point.id), entry_point.token)))
+                    rval = (url_for('%s/%s/%s/%s/' % (trans.app.config.realtime_prefix, entry_point.__class__.__name__, trans.security.encode_id(entry_point.id), entry_point.token)))
                     if entry_point.entry_url:
                         rval = '%s%s' % (rval, entry_point.entry_url)
                     return trans.response.send_redirect(rval)
@@ -137,12 +126,12 @@ class RealTime(BaseUIController):
     def stop(self, trans, realtime_id=None, realtime_entry_id=None):
         """List all available realtimetools"""
         if realtime_id:
-            rtt = trans.app.model.RealTimeTool.get(trans.security.decode_id(realtime_id))
+            realtime = trans.app.model.RealTimeTool.get(trans.security.decode_id(realtime_id))
         elif realtime_entry_id:
-            rtt = trans.app.model.RealTimeToolEntryPoint.get(trans.security.decode_id(realtime_entry_id)).realtime
+            realtime = trans.app.model.RealTimeToolEntryPoint.get(trans.security.decode_id(realtime_entry_id)).realtime
         else:
-            return dict(message="Can't stop: RealTimeTool not provided.",state='error')
+            return dict(message="Can't stop: RealTimeTool not provided.", state='error')
         stopped = trans.app.realtime_manager.stop(realtime)
         if stopped:
-            return dict(message="RealTimeTool has been stopped",state='ok')
-        return dict(message="Unable to stop RealTimeTool",state='error')
+            return dict(message="RealTimeTool has been stopped", state='ok')
+        return dict(message="Unable to stop RealTimeTool", state='error')

From 22f6d48eaec0e6e58be2b89c16e00f9b946ca47a Mon Sep 17 00:00:00 2001
From: Nicola Soranzo <nicola.soranzo@gmail.com>
Date: Fri, 8 Mar 2019 10:59:05 -0800
Subject: [PATCH 003/114] Update lib/galaxy/datatypes/realtime.py

Co-Authored-By: blankenberg <dan.blankenberg@gmail.com>
---
 lib/galaxy/datatypes/realtime.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/datatypes/realtime.py b/lib/galaxy/datatypes/realtime.py
index 63df3956c35..e4468232198 100644
--- a/lib/galaxy/datatypes/realtime.py
+++ b/lib/galaxy/datatypes/realtime.py
@@ -5,7 +5,7 @@ import logging
 import os
 import sys
 
-from text import Html
+from .text import Html
 from galaxy.web import url_for
 
 

From a7c9ac042f41c9b0b416875b8cc1c676b5af30b9 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Sat, 9 Mar 2019 15:56:16 -0500
Subject: [PATCH 004/114] fix

---
 config/galaxy.yml.realtime                        | 11 +++--------
 lib/galaxy/managers/realtime.py                   |  2 +-
 lib/galaxy/webapps/galaxy/controllers/realtime.py |  3 ++-
 3 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/config/galaxy.yml.realtime b/config/galaxy.yml.realtime
index 25eeaa589ff..bffc0bd1d28 100644
--- a/config/galaxy.yml.realtime
+++ b/config/galaxy.yml.realtime
@@ -14,19 +14,14 @@ uwsgi:
 
   sessions: database/session_map.sqlite
   python-raw: lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
-  route-run: log:dynamic_proxy_prefix ${dynamic_proxy_prefix}
-  route: ^/realtime-proxy/access/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)(()|(/.*)*)$ goto:realtime
+  route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.localhost:8080)$ goto:realtime
   route-run: goto:endendend
   route-label: realtime
-  route: ^/realtime-proxy/access/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)(()|/.*)$ rpcvar:TARGET_HOST rtt_key_type_token_mapper $2 $1 $3 $4 $0
-  route: ^/realtime-proxy/access/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)/([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)((()|/.*)*)$ rewrite:$4
-  route-if: empty:${PATH_INFO} addvar:PATH_INFO=/
-  route-run: seturi:${PATH_INFO}
-  route-if-not: empty:${QUERY_STRING} seturi:${PATH_INFO}?${QUERY_STRING}
+  route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.localhost:8080)$ rpcvar:TARGET_HOST rtt_key_type_token_mapper $2 $1 $3 $4 $0
   route-if-not: empty:${TARGET_HOST} httpdumb:${TARGET_HOST}
   route-label: endendend
 
 
 galaxy:
 
-  realtime_prefix: /realtime-proxy/access
+  realtime_prefix: realtime.localhost
diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index f7d3a206b0e..db66be71e55 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -90,7 +90,7 @@ class RealtimeSqlite(object):
     def save_entry_point(self, entry_point):
         """Convenience method to easily save an entry_point.
         """
-        return self.save(self.encode_id(entry_point.id), entry_point.__class__.__name__, entry_point.token, entry_point.host, entry_point.port, None)
+        return self.save(self.encode_id(entry_point.id), entry_point.__class__.__name__.lower(), entry_point.token, entry_point.host, entry_point.port, None)
 
 
 class RealTimeManager(object):
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index 1b843449d8a..451344718a8 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -87,7 +87,8 @@ class RealTime(BaseUIController):
             entry_point = trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).get(entry_point_id)
             if trans.app.realtime_manager.can_access_entry_point(trans, entry_point):
                 if entry_point.realtime.active and entry_point.configured:
-                    rval = (url_for('%s/%s/%s/%s/' % (trans.app.config.realtime_prefix, entry_point.__class__.__name__, trans.security.encode_id(entry_point.id), entry_point.token)))
+                    rval = '%s//%s.%s.%s.%s.%s/' % (trans.request.host_url.split('//', 1)[0], entry_point.__class__.__name__.lower(), trans.security.encode_id(entry_point.id),
+                            entry_point.token, trans.app.config.realtime_prefix, trans.request.host)
                     if entry_point.entry_url:
                         rval = '%s%s' % (rval, entry_point.entry_url)
                     return trans.response.send_redirect(rval)

From b41377134ef02aad0a002d3414f2572a875485b1 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Sat, 9 Mar 2019 15:58:38 -0500
Subject: [PATCH 005/114] fix

---
 config/galaxy.yml.realtime | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/galaxy.yml.realtime b/config/galaxy.yml.realtime
index bffc0bd1d28..a55c3439ff4 100644
--- a/config/galaxy.yml.realtime
+++ b/config/galaxy.yml.realtime
@@ -24,4 +24,4 @@ uwsgi:
 
 galaxy:
 
-  realtime_prefix: realtime.localhost
+  realtime_prefix: realtime

From acbee59546484758a3dc2dcf60d6c00dabfcc5ab Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Wed, 13 Mar 2019 17:38:15 -0400
Subject: [PATCH 006/114] Make stop button in Active RealTimeTool list do
 something.

---
 lib/galaxy/jobs/runners/local.py              | 24 +++++---
 lib/galaxy/managers/realtime.py               | 18 +++++-
 .../webapps/galaxy/controllers/realtime.py    | 58 +++++++++++--------
 3 files changed, 65 insertions(+), 35 deletions(-)

diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py
index 144c851c8f5..b457a20e972 100644
--- a/lib/galaxy/jobs/runners/local.py
+++ b/lib/galaxy/jobs/runners/local.py
@@ -111,6 +111,8 @@ class LocalJobRunner(BaseJobRunner):
                 job_wrapper.set_job_destination(job_wrapper.job_destination, proc.pid)
                 job_wrapper.change_state(model.Job.states.RUNNING)
 
+                self.__handle_container(job_wrapper, proc)
+
                 terminated = self.__poll_if_needed(proc, job_wrapper, job_id)
                 if terminated:
                     return
@@ -238,6 +240,8 @@ class LocalJobRunner(BaseJobRunner):
             return False
 
     def _terminate(self, proc):
+        if proc.container_commands:
+            self._run_command(proc.container_commands['kill'])
         os.killpg(proc.pid, 15)
         sleep(1)
         if proc.poll() is None:
@@ -256,13 +260,18 @@ class LocalJobRunner(BaseJobRunner):
             cont = job.container
             if cont:
                 if cont.container_type == 'docker':
-                    exit_code = subprocess.call(cont.container_info['commands'][command],
-                                            shell=True,
-                                            env=self._environ,
-                                            preexec_fn=os.setpgrp)
-                    log.debug('docker container command exit code %s', exit_code)
+                    return self._run_command(cont.container_info['commands'][command])
 
-    def __handle_container(self, job_wrapper):
+    def _run_command(self, command):
+        exit_code = subprocess.call(command,
+                                    shell=True,
+                                    env=self._environ,
+                                    preexec_fn=os.setpgrp)
+        log.debug('_run_command(%s) exit code %s', command, exit_code)
+        return exit_code
+
+    def __handle_container(self, job_wrapper, proc):
+        proc.container_commands = None
         job = job_wrapper.get_job()
         if job:
             cont = job.container
@@ -315,10 +324,10 @@ class LocalJobRunner(BaseJobRunner):
                                 # otherwise, Galaxy's db doesn't get written to, but the sqlite db does
                                 job_wrapper.sa_session.add(ep)
                                 job_wrapper.sa_session.flush()
+                    proc.container_commands = cont.container_info['commands']
 
     def __poll_if_needed(self, proc, job_wrapper, job_id):
         # Only poll if needed (i.e. job limits are set)
-        self.__handle_container(job_wrapper)
         if not job_wrapper.has_limits():
             return
 
@@ -331,7 +340,6 @@ class LocalJobRunner(BaseJobRunner):
                 limit_state = job_wrapper.check_limits(runtime=datetime.datetime.now() - job_start)
                 if limit_state is not None:
                     job_wrapper.fail(limit_state[1])
-                    self.__kill_container(job_wrapper)
                     log.debug('(%s) Terminating process group' % job_id)
                     self._terminate(proc)
                     return True
diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index db66be71e55..1af91335d45 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -101,6 +101,7 @@ class RealTimeManager(object):
     def __init__(self, app):
         self.model = app.model
         self.security = app.security
+        self.job_manager = app.job_manager
         self.propagator = RealtimeSqlite(app.config.proxy_session_map, app.security.encode_id)
 
     # FIXME: Do we really want to directly associate a single dataset?
@@ -191,5 +192,18 @@ class RealTimeManager(object):
             return self.can_access_realtime(trans, entry_point.realtime)
         return False
 
-    def stop(self, realtime):
-        raise Exception('not implemented')
+    def stop(self, trans, realtime):
+        try:
+            job = realtime.job
+            if not job.finished:
+                # FIXME: Need stop container, but set to job OK.
+                log.debug('Stopping Job: %s for RealTimeTool: %s', job, realtime)
+                job.mark_deleted(trans.app.config.track_jobs_in_database)
+                # This self.job_manager.stop(job) does nothing without changing job.state, manually or e.g. with .mark_deleted()
+                self.job_manager.stop(job)
+                trans.sa_session.add(job)
+                trans.sa_session.flush()
+        except Exception as e:
+            log.debug('Unable to stop job for RealTimeTool: %s', e)
+            return False
+        return True
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index 451344718a8..70640d696f0 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -47,7 +47,6 @@ class RealTimeToolEntryPointListGrid(grids.Grid):
         )
     )
     operations = [
-        grids.GridOperation("Open", condition=(lambda item: item.active), async_compatible=False),
         grids.GridOperation("Stop", condition=(lambda item: item.active), async_compatible=False),
     ]
 
@@ -96,32 +95,41 @@ class RealTime(BaseUIController):
                     return trans.show_error_message('RealTimeTool is not active. If you recently launched this tool it may not be ready yet, please wait a moment and refresh this page.')
         return trans.show_error_message('Access not authorized.')
 
-    def _get_job_for_dataset(self, trans, dataset_id):
-        '''
-        Return the job for the given dataset. This will throw an error if the
-        dataset is either nonexistent or inaccessible to the user.
-        '''
-        hda = trans.sa_session.query(trans.app.model.HistoryDatasetAssociation).get(self.decode_id(dataset_id))
-        return hda.creating_job
-
-    def _get_job(self, trans, job_id):
-        '''
-        Return the job for the given dataset. This will throw an error if the
-        dataset is either nonexistent or inaccessible to the user.
-        '''
-        return trans.sa_session.query(trans.app.model.Job).get(self.decode_id(job_id))
-
     @web.expose_api
     def list(self, trans, **kwargs):
         """List all available realtimetools"""
-        return self._rtt_ep_grid(trans)
-        rtts = trans.app.realtime_manager.get_nonterminal_for_user_by_trans(trans)
-        eps = []
-        if rtts:
-            for rtt in rtts:
-                for ep in rtt.entry_points:
-                    eps.append(self.encode_all_ids(trans, ep.to_dict(), True))
-        return self._rtt_ep_grid(eps)
+        log.debug('list kargs %s', str(kwargs))
+        operation = kwargs.get('operation', None)
+        message = None
+        if operation:
+            eps = []
+            rtts = []
+            ids = kwargs.get('id', None)
+            if ids:
+                if not isinstance(ids, list):
+                    ids = [ids]
+                for entry_point_id in ids:
+                    entry_point_id = self.decode_id(entry_point_id)
+                    entry_point = trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).get(entry_point_id)
+                    if trans.app.realtime_manager.can_access_entry_point(trans, entry_point):
+                        eps.append(entry_point)
+                        if entry_point.realtime not in rtts:
+                            rtts.append(entry_point.realtime)
+            if rtts:
+                failed = []
+                succeeded = []
+                if operation == 'stop':
+                    for realtime in rtts:
+                        stopped = trans.app.realtime_manager.stop(trans, realtime)
+                        if stopped:
+                            succeeded.append(realtime)
+                        else:
+                            failed.append(realtime)
+                    if failed:
+                        message = 'Unable to stop %i RealTimeTools.' % (len(failed))
+                    if succeeded:
+                        message = 'Stopped %i RealTimeTools.' % (len(succeeded))
+        return self._rtt_ep_grid(trans, message=message)
 
     @web.expose_api
     def stop(self, trans, realtime_id=None, realtime_entry_id=None):
@@ -132,7 +140,7 @@ class RealTime(BaseUIController):
             realtime = trans.app.model.RealTimeToolEntryPoint.get(trans.security.decode_id(realtime_entry_id)).realtime
         else:
             return dict(message="Can't stop: RealTimeTool not provided.", state='error')
-        stopped = trans.app.realtime_manager.stop(realtime)
+        stopped = trans.app.realtime_manager.stop(trans, realtime)
         if stopped:
             return dict(message="RealTimeTool has been stopped", state='ok')
         return dict(message="Unable to stop RealTimeTool", state='error')

From 593bffbeb45e8a8e5abfe19f867ef4a064344464 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Thu, 14 Mar 2019 13:36:14 -0400
Subject: [PATCH 007/114] better handling of missing containers

---
 lib/galaxy/jobs/runners/local.py | 52 +++++++++++++++++++-------------
 1 file changed, 31 insertions(+), 21 deletions(-)

diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py
index b457a20e972..12013bc15c8 100644
--- a/lib/galaxy/jobs/runners/local.py
+++ b/lib/galaxy/jobs/runners/local.py
@@ -278,8 +278,10 @@ class LocalJobRunner(BaseJobRunner):
             if cont:
                 if cont.container_type == 'docker':
                     # Do we need to do inspect, or is only ports ok?
+                    max_command_attempts = 10 + 1
                     inspect = None
-                    for i in range(1, 11):
+                    exit_code = None
+                    for i in range(1, max_command_attempts):
                         with tempfile.TemporaryFile() as stdout_file:
                             exit_code = subprocess.call(cont.container_info['commands']['inspect'],
                                                     shell=True,
@@ -290,14 +292,18 @@ class LocalJobRunner(BaseJobRunner):
                                 stdout_file.seek(0)
                                 inspect = stdout_file.read()
                                 break
-                        t = 2 * i
-                        log.debug("Container not found, sleeping for %s seconds.", t)
-                        sleep(t)
-                    cont.container_info['inspect'] = json.loads(inspect)
+                        if i != max_command_attempts:
+                            t = 2 * i
+                            log.debug("Container not found during inspect check, sleeping for %s seconds.", t)
+                            sleep(t)
+                    if inspect:
+                        cont.container_info['inspect'] = json.loads(inspect)
+                    else:
+                        log.error('Unable to run inspect command (%s): %s', cont.container_info['commands']['inspect'], exit_code)
                     rtt = job.realtime_tool
                     if rtt:
                         ports_raw = None
-                        for i in range(1, 11):
+                        for i in range(1, max_command_attempts):
                             with tempfile.TemporaryFile() as stdout_file:
                                 exit_code = subprocess.call(cont.container_info['commands']['port'],
                                                         shell=True,
@@ -308,22 +314,26 @@ class LocalJobRunner(BaseJobRunner):
                                     stdout_file.seek(0)
                                     ports_raw = stdout_file.read()
                                     break
-                            t = 2 * i
-                            log.debug("Container not found, sleeping for %s seconds.", t)
-                            sleep(t)
+                            if i != max_command_attempts:
+                                t = 2 * i
+                                log.debug("Container not found during port check, sleeping for %s seconds.", t)
+                                sleep(t)
 
-                        for line in ports_raw.strip().split('\n'):
-                            # TODO: configure all ports at once, instead of individually
-                            # A port could e.g. be mapped multiple times, with diff entry URL
-                            tool, host = line.split(" -> ", 1)
-                            hostname, port = host.split(':')
-                            tool_p, tool_prot = tool.split("/")
-                            ep = self.app.realtime_manager.configure_entry_point(rtt, tool_port=tool_p, host=hostname, port=port, protocol=tool_prot)
-                            if ep:
-                                # must add and flush the ep, here or elsewhere
-                                # otherwise, Galaxy's db doesn't get written to, but the sqlite db does
-                                job_wrapper.sa_session.add(ep)
-                                job_wrapper.sa_session.flush()
+                        if ports_raw is not None:
+                            for line in ports_raw.strip().split('\n'):
+                                # TODO: configure all ports at once, instead of individually
+                                # A port could e.g. be mapped multiple times, with diff entry URL
+                                tool, host = line.split(" -> ", 1)
+                                hostname, port = host.split(':')
+                                tool_p, tool_prot = tool.split("/")
+                                ep = self.app.realtime_manager.configure_entry_point(rtt, tool_port=tool_p, host=hostname, port=port, protocol=tool_prot)
+                                if ep:
+                                    # must add and flush the ep, here or elsewhere
+                                    # otherwise, Galaxy's db doesn't get written to, but the sqlite db does
+                                    job_wrapper.sa_session.add(ep)
+                                    job_wrapper.sa_session.flush()
+                        else:
+                            log.error('Unable to run port command (%s): %s', cont.container_info['commands']['port'], exit_code)
                     proc.container_commands = cont.container_info['commands']
 
     def __poll_if_needed(self, proc, job_wrapper, job_id):

From 054f1fc9d3ebb26b66af0acc40c3002dc1e5218f Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 15 Mar 2019 14:21:23 -0400
Subject: [PATCH 008/114] Update access to url_for().

---
 lib/galaxy/tools/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 3f5d4bc3a93..b0bb35533f9 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -2327,7 +2327,7 @@ class RealTimeTool(Tool):
         if job:
             realtimetool = job.realtime_tool
             if realtimetool:
-                return url_for(controller='realtime', action='index', realtime_id=self.app.security.encode_id(realtimetool.id))
+                return self.app.url_for(controller='realtime', action='index', realtime_id=self.app.security.encode_id(realtimetool.id))
         return None
 
 

From 696c2cf1868d6af8da293f1d18c52f19bc808bc8 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 15 Mar 2019 15:54:10 -0400
Subject: [PATCH 009/114] import ordering

---
 lib/galaxy/datatypes/realtime.py                      | 2 +-
 lib/galaxy/tools/__init__.py                          | 2 +-
 lib/galaxy/tools/deps/container_classes.py            | 5 ++---
 lib/galaxy/web/realtime/rpc/key_type_token_mapping.py | 3 ++-
 lib/galaxy/webapps/galaxy/controllers/realtime.py     | 1 -
 5 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/lib/galaxy/datatypes/realtime.py b/lib/galaxy/datatypes/realtime.py
index e4468232198..dfdf6018211 100644
--- a/lib/galaxy/datatypes/realtime.py
+++ b/lib/galaxy/datatypes/realtime.py
@@ -5,8 +5,8 @@ import logging
 import os
 import sys
 
-from .text import Html
 from galaxy.web import url_for
+from .text import Html
 
 
 log = logging.getLogger(__name__)
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index b0bb35533f9..28dea202848 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -32,11 +32,11 @@ from galaxy.queue_worker import send_control_task
 from galaxy.tools.actions import DefaultToolAction
 from galaxy.tools.actions.data_manager import DataManagerToolAction
 from galaxy.tools.actions.data_source import DataSourceToolAction
-from galaxy.tools.actions.realtime import RealTimeToolAction
 from galaxy.tools.actions.model_operations import ModelOperationToolAction
 from galaxy.tools.deps import (
     CachedDependencyManager,
 )
+from galaxy.tools.actions.realtime import RealTimeToolAction
 from galaxy.tools.fetcher import ToolLocationFetcher
 from galaxy.tools.parameters import (
     check_param,
diff --git a/lib/galaxy/tools/deps/container_classes.py b/lib/galaxy/tools/deps/container_classes.py
index c442f95eb34..82e1f1d61c8 100644
--- a/lib/galaxy/tools/deps/container_classes.py
+++ b/lib/galaxy/tools/deps/container_classes.py
@@ -4,13 +4,12 @@ from abc import (
     ABCMeta,
     abstractmethod
 )
-
 from uuid import uuid4
 
-import six
-
 from logging import getLogger
 
+import six
+
 from galaxy.containers.docker_model import DockerVolume
 from galaxy.util import (
     asbool,
diff --git a/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py b/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
index 0b9217a3e2c..9d26834de5a 100644
--- a/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
+++ b/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
@@ -1,6 +1,7 @@
-import uwsgi
 import sqlite3
 
+import uwsgi
+
 
 realtime_db_file = uwsgi.opt["sessions"]
 db_conn = sqlite3.connect(realtime_db_file)
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index 70640d696f0..72ef2717e48 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -11,7 +11,6 @@ from galaxy.web import url_for
 from galaxy.web.base.controller import (
     BaseUIController,
 )
-
 from galaxy.web.framework.helpers import (
     grids,
     time_ago,

From dcdbee1ec2e063952418f23d617e3917a2c4ff9e Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 15 Mar 2019 21:19:27 -0400
Subject: [PATCH 010/114] Only display view result message when view is
 available.

---
 client/galaxy/scripts/mvc/tool/tool-form.js | 25 ++++++++++++---------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js
index 94f108e60b4..8a2170db72e 100644
--- a/client/galaxy/scripts/mvc/tool/tool-form.js
+++ b/client/galaxy/scripts/mvc/tool/tool-form.js
@@ -347,19 +347,22 @@ var View = Backbone.View.extend({
     },
 
     _getViewResult: function(response) {
+        var vr = "";
         if (response.view_result) {
-            return `<div class="infomessagelarge">
-                        <p>
-                            <a href="${response.view_result}">
-                            There is a RealTimeTool result view available, click here to display. You can also click on the display eye icon (<span class="fa fa-eye"></span>)
-                            associated with the RealTimeTool Dataset to view.</a>
-                        </p>
-                        <p>
-                            You may also access all active RealTimeTools from the User menu.
-                        </p>
-                    </div>`;
+            for (let i in response.view_result) {
+                vr += `<div class="infomessagelarge">
+                            <p>
+                                <a href="${response.view_result[i]}">
+                                There is a RealTimeTool result view available, click here to display. You can also click on the display eye icon (<span class="fa fa-eye"></span>)
+                                associated with the RealTimeTool Dataset to view.</a>
+                            </p>
+                            <p>
+                                You may also access all active RealTimeTools from the User menu.
+                            </p>
+                        </div>`;
+            }
         }
-        return "";
+        return vr;
     },
 
     _templateRow: function(list, title, max = 3) {

From 2bf465d57cd7eeafdd53051723c549ad5576303a Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Wed, 20 Mar 2019 11:20:19 -0400
Subject: [PATCH 011/114] Add a ttl cache aware mapper to
 key_type_token_mapping.py.

---
 .../realtime/rpc/key_type_token_mapping.py    | 56 +++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py b/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
index 9d26834de5a..4471b38638e 100644
--- a/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
+++ b/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
@@ -1,5 +1,9 @@
+import itertools
 import sqlite3
 
+from threading import RLock
+from time import time
+
 import uwsgi
 
 
@@ -7,6 +11,57 @@ realtime_db_file = uwsgi.opt["sessions"]
 db_conn = sqlite3.connect(realtime_db_file)
 
 
+class CacheEntry():
+    def __init__(self, key, value, ttl=20):
+        self.key = key
+        self.value = value
+        self.expires_at = time() + ttl
+        self._expired = False
+
+    def expired(self):
+        if self._expired is False:
+            return (self.expires_at < time())
+        else:
+            return self._expired
+
+
+class CacheList():
+    def __init__(self):
+        self.entries = []
+        self.lock = RLock()
+
+    def add_entry(self, key, value, ttl=20):
+        with self.lock:
+            self.entries.append(CacheEntry(key, value, ttl))
+
+    def read_entries(self):
+        with self.lock:
+            self.entries = list(itertools.dropwhile(lambda x: x.expired(), self.entries))
+            return self.entries
+
+    def get_entry_value(self, key, default=None):
+        entries = self.read_entries()
+        for entry in entries:
+            if entry.key == key:
+                return entry.value
+        return default
+
+
+key_type_token_mapped_cache = CacheList()
+
+
+def key_type_token_mapper_cached(key, key_type, token, route_extra, url, ttl):
+    global key_type_token_mapped_cache
+    cache_key = (key, key_type, token)
+    entry = key_type_token_mapped_cache.get_entry_value(cache_key, None)
+    if entry is None:
+        entry = key_type_token_mapper(key, key_type, token, route_extra, url)
+        if entry is not None:
+            # Should we cache empt/not authorized entries, perhaps for shorter time?
+            key_type_token_mapped_cache.add_entry(cache_key, entry, ttl=float(ttl))
+    return entry
+
+
 def key_type_token_mapper(key, key_type, token, route_extra, url):
     global db_conn
     # print 'key %s key_type %s token %s route_extra %s url %s\n' % (key, key_type, token, route_extra, url)
@@ -29,3 +84,4 @@ def key_type_token_mapper(key, key_type, token, route_extra, url):
 
 
 uwsgi.register_rpc('rtt_key_type_token_mapper', key_type_token_mapper)
+uwsgi.register_rpc('rtt_key_type_token_mapper_cached', key_type_token_mapper_cached)

From 2570cb179b7050765ff08a4a9fe5f6aa9b49e168 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Wed, 20 Mar 2019 12:47:37 -0400
Subject: [PATCH 012/114] Remove entries from realtimetool sqlite db when job
 has ended.

---
 lib/galaxy/managers/realtime.py | 45 +++++++++++++++++++++++++++++++++
 lib/galaxy/tools/__init__.py    | 19 ++++++++++++++
 2 files changed, 64 insertions(+)

diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index 1af91335d45..9aca8c5028a 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -87,11 +87,49 @@ class RealtimeSqlite(object):
             finally:
                 conn.close()
 
+    def remove(self, **kwd):
+        """
+        Remove entry from a key, key_type, token, value store that is can be used for coordinating
+        with external resources. Remove entries that match all provided key=values
+        """
+        assert kwd, ValueError("You must provide some values to key upon")
+        delete = 'DELETE FROM %s WHERE' % (DATABASE_TABLE_NAME)
+        value_list = []
+        for i, (key, value) in enumerate(kwd.items()):
+            if i != 0:
+                delete += ' and'
+            delete += ' %s=?' % (key)
+            value_list.append(value)
+        with FileLock(self.sqlite_filename):
+            conn = sqlite3.connect(self.sqlite_filename)
+            try:
+                c = conn.cursor()
+                try:
+                    # Delete entry
+                    # NB: This does not invalidate in-memory caches used by uwsgi (if any)
+                    c.execute(delete, tuple(value_list))
+                except Exception as e:
+                    log.debug('Error removing entry (%s): %s', delete, e)
+                conn.commit()
+            finally:
+                conn.close()
+
     def save_entry_point(self, entry_point):
         """Convenience method to easily save an entry_point.
         """
         return self.save(self.encode_id(entry_point.id), entry_point.__class__.__name__.lower(), entry_point.token, entry_point.host, entry_point.port, None)
 
+    def remove_entry_point(self, entry_point):
+        """Convenience method to easily remove an entry_point.
+        """
+        return self.remove(key=self.encode_id(entry_point.id), key_type=entry_point.__class__.__name__.lower())
+
+    def remove_realtime(self, rtt):
+        """Convenience method to easily remove a RealTimeTool.
+        """
+        for ep in rtt.entry_points:
+            self.remove_entry_point(ep)
+
 
 class RealTimeManager(object):
     """
@@ -194,6 +232,7 @@ class RealTimeManager(object):
 
     def stop(self, trans, realtime):
         try:
+            self.remove_realtime(realtime)
             job = realtime.job
             if not job.finished:
                 # FIXME: Need stop container, but set to job OK.
@@ -207,3 +246,9 @@ class RealTimeManager(object):
             log.debug('Unable to stop job for RealTimeTool: %s', e)
             return False
         return True
+
+    def remove_realtime(self, realtime):
+        return self.propagator.remove_realtime(realtime)
+
+    def remove_entry_point(self, entry_point):
+        return self.propagator.remove_entry_point(entry_point)
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 28dea202848..85dc9fe3d1f 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -2323,6 +2323,25 @@ class RealTimeTool(Tool):
     tool_type = 'realtime'
     default_tool_action = RealTimeToolAction
 
+    def __remove_realtime_by_job(self, job):
+        if job:
+            rtt = job.realtime_tool
+            log.debug('__remove_realtime_by_job: %s', rtt)
+            if rtt:
+                self.app.realtime_manager.remove_realtime(rtt)
+        else:
+            log.warning("Could not determine job to stop RealTimeTool: %s", job)
+
+    def exec_after_process(self, app, inp_data, out_data, param_dict, job=None, **kwds):
+        # run original exec_after_process
+        super(RealTimeTool, self).exec_after_process(app, inp_data, out_data, param_dict, job=job, **kwds)
+        self.__remove_realtime_by_job(job)
+
+    def job_failed(self, job_wrapper, message, exception=False):
+        super(RealTimeTool, self).job_failed(job_wrapper, message, exception=exception)
+        job = job_wrapper.sa_session.query(model.Job).get(job_wrapper.job_id)
+        self.__remove_realtime_by_job(job)
+
     def get_view_result(self, job=None):
         if job:
             realtimetool = job.realtime_tool

From 9274f7db60585e0773cea0b971ac2b538869c24f Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Wed, 20 Mar 2019 17:29:10 -0400
Subject: [PATCH 013/114] Simplify RealTimeTool model, associated entry points
 directly with job.

---
 lib/galaxy/datatypes/realtime.py              |  12 +-
 lib/galaxy/jobs/runners/local.py              |  17 +--
 lib/galaxy/managers/realtime.py               | 122 +++++++++++-------
 lib/galaxy/model/__init__.py                  |  31 ++---
 lib/galaxy/model/mapping.py                   |  27 +---
 .../versions/0149_add_realtimetools.py        |  37 ++----
 lib/galaxy/tools/__init__.py                  |  13 +-
 lib/galaxy/tools/actions/realtime.py          |   2 +-
 lib/galaxy/tools/deps/docker_util.py          |  14 ++
 .../webapps/galaxy/controllers/realtime.py    |  54 ++++----
 10 files changed, 157 insertions(+), 172 deletions(-)

diff --git a/lib/galaxy/datatypes/realtime.py b/lib/galaxy/datatypes/realtime.py
index dfdf6018211..1cc9847affd 100644
--- a/lib/galaxy/datatypes/realtime.py
+++ b/lib/galaxy/datatypes/realtime.py
@@ -31,8 +31,7 @@ class Realtime(Html):
 
     def generate_primary_file(self, dataset=None):
         """
-        This is called only at upload to write the html file
-        cannot rename the datasets here - they come with the default unfortunately
+        This is called only at creation to write the html file
         """
         defined_files = self.get_composite_files(dataset=dataset).items()
         rval = ["<html><head><title>Files for Composite Dataset (%s)</title></head>" % (self.file_ext)]
@@ -65,10 +64,11 @@ class Realtime(Html):
         return "\n".join(rval)
 
     def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, offset=None, ck_size=None, **kwd):
-        log.debug('dataset.realtime_tool.id %s', dataset.realtime_tool.id)
-        rtt = dataset.realtime_tool
-        if rtt and rtt.active:
-            return trans.response.send_redirect(url_for(controller='realtime', action='index', realtime_id=trans.security.encode_id(rtt.id)))
+        job = dataset.creating_job
+        if job and not job.finished:
+            eps = job.realtimetool_entry_points
+            if eps:
+                return trans.response.send_redirect(url_for(controller='realtime', action='index', entry_point_id=[trans.security.encode_id(ep.id) for ep in eps]))
         return super(Realtime, self).display_data(trans, dataset, preview=preview, filename=filename, to_ext=to_ext, offset=offset, ck_size=ck_size, **kwd)
 
 
diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py
index 24828354f7c..9b87296ea44 100644
--- a/lib/galaxy/jobs/runners/local.py
+++ b/lib/galaxy/jobs/runners/local.py
@@ -286,8 +286,8 @@ class LocalJobRunner(BaseJobRunner):
                         cont.container_info['inspect'] = json.loads(inspect)
                     else:
                         log.error('Unable to run inspect command (%s): %s', cont.container_info['commands']['inspect'], exit_code)
-                    rtt = job.realtime_tool
-                    if rtt:
+                    eps = job.realtimetool_entry_points
+                    if eps:
                         ports_raw = None
                         for i in range(1, max_command_attempts):
                             with tempfile.TemporaryFile() as stdout_file:
@@ -306,18 +306,7 @@ class LocalJobRunner(BaseJobRunner):
                                 sleep(t)
 
                         if ports_raw is not None:
-                            for line in ports_raw.strip().split('\n'):
-                                # TODO: configure all ports at once, instead of individually
-                                # A port could e.g. be mapped multiple times, with diff entry URL
-                                tool, host = line.split(" -> ", 1)
-                                hostname, port = host.split(':')
-                                tool_p, tool_prot = tool.split("/")
-                                ep = self.app.realtime_manager.configure_entry_point(rtt, tool_port=tool_p, host=hostname, port=port, protocol=tool_prot)
-                                if ep:
-                                    # must add and flush the ep, here or elsewhere
-                                    # otherwise, Galaxy's db doesn't get written to, but the sqlite db does
-                                    job_wrapper.sa_session.add(ep)
-                                    job_wrapper.sa_session.flush()
+                            self.app.realtime_manager.configure_entry_points_raw_docker_ports(job, ports_raw)
                         else:
                             log.error('Unable to run port command (%s): %s', cont.container_info['commands']['port'], exit_code)
                     proc.container_commands = cont.container_info['commands']
diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index 9aca8c5028a..44acf51a246 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -6,12 +6,12 @@ from six import string_types
 from sqlalchemy import or_
 
 
+from galaxy.tools.deps.docker_util import parse_port_text as docker_parse_port_text
 from galaxy.util.filelock import FileLock
 
 
 log = logging.getLogger(__name__)
 
-REALTIMETOOL_DATASET_EXT = 'realtimetool'
 DATABASE_TABLE_NAME = 'gxrtproxy'
 
 
@@ -124,7 +124,15 @@ class RealtimeSqlite(object):
         """
         return self.remove(key=self.encode_id(entry_point.id), key_type=entry_point.__class__.__name__.lower())
 
-    def remove_realtime(self, rtt):
+    def remove_entry_points0(self, entry_points):
+        """Convenience method to easily remove entry_points.
+        """
+        rval = []
+        for entry_point in entry_points:
+            rval.append(self.remove_entry_point(entry_point))
+        return rval
+
+    def remove_realtime0(self, rtt):
         """Convenience method to easily remove a RealTimeTool.
         """
         for ep in rtt.entry_points:
@@ -139,42 +147,45 @@ class RealTimeManager(object):
     def __init__(self, app):
         self.model = app.model
         self.security = app.security
+        self.sa_session = app.model.context
         self.job_manager = app.job_manager
         self.propagator = RealtimeSqlite(app.config.proxy_session_map, app.security.encode_id)
 
-    # FIXME: Do we really want to directly associate a single dataset?
-    # Probably Not
-    def guess_associated_dataset(self, datasets):
-        for dataset in datasets:
-            dataset = dataset.dataset
-            if dataset.ext == REALTIMETOOL_DATASET_EXT:
-                return dataset
-        return None
-
-    def create_entry_points(self, trans, rtt, tool, flush=True):
+    def create_entry_points(self, job, tool, flush=True):
         for entry in tool.tool_ports:
-            rtt_entry = self.model.RealTimeToolEntryPoint(realtime=rtt, tool_port=entry['port'], entry_url=entry['url'], name=entry['name'])
-            trans.sa_session.add(rtt_entry)
+            ep = self.model.RealTimeToolEntryPoint(job=job, tool_port=entry['port'], entry_url=entry['url'], name=entry['name'])
+            self.sa_session.add(ep)
         if flush:
-            trans.sa_session.flush()
+            self.sa_session.flush()
 
-    def configure_entry_point(self, rtt, tool_port=None, host=None, port=None, protocol=None):
-        if tool_port is not None:
-            tool_port = int(tool_port)
+    def configure_entry_point(self, job, tool_port=None, host=None, port=None, protocol=None):
+        return self.configure_entry_points(job, {tool_port: dict(tool_port=tool_port, host=host, port=port, protocol=protocol)})
+
+    def configure_entry_points(self, job, ports_dict):
+        # There can be multiple entry points that reference the same tool port (could have different entry URLs)
+        configured = []
         not_configured = []
-        for entry in rtt.entry_points:
-            if entry.tool_port == tool_port:
-                entry.host = host
-                entry.port = port
-                entry.protocol = protocol
-                entry.configured = True
-                self.save_entry_point(entry)
-                return entry
-            elif not entry.configured:
-                not_configured.append(entry)
-        # TODO: port not found in entry, add to random one?
-        log.warning('tool port not found! %s', str(not_configured))
-        return None
+        log.debug('ports_dict %s', str(ports_dict))
+        for ep in job.realtimetool_entry_points:
+            port_dict = ports_dict.get(ep.tool_port, None)
+            if port_dict is None:
+                log.error("Did not find port to assign to RealTimeToolEntryPoint by tool port: %s.", ep.tool_port)
+                not_configured.append(ep)
+            else:
+                ep.host = port_dict['host']
+                ep.port = port_dict['port']
+                ep.protocol = port_dict['protocol']
+                ep.configured = True
+                self.sa_session.add(ep)
+                self.save_entry_point(ep)
+                configured.append(ep)
+        if configured:
+            self.sa_session.flush()
+        return dict(not_configured=not_configured, configured=configured)
+
+    def configure_entry_points_raw_docker_ports(self, job, port_text):
+        ports_dict = docker_parse_port_text(port_text)
+        return self.configure_entry_points(job, ports_dict)
 
     def save_entry_point(self, entry_point):
         """
@@ -182,16 +193,12 @@ class RealTimeManager(object):
         """
         self.propagator.save_entry_point(entry_point)
 
-    def create_realtime(self, trans, job, tool):
+    def create_realtime(self, job, tool):
         # create from initial job
-        if job:
-            realtime_tool = self.model.RealTimeTool(job=job, user=trans.user, session=trans.galaxy_session, dataset=self.guess_associated_dataset(job.get_output_datasets()))
-            trans.sa_session.add(realtime_tool)
-            if tool:
-                self.create_entry_points(trans, realtime_tool, tool, flush=False)
-            trans.sa_session.flush()
+        if job and tool:
+            self.create_entry_points(job, tool)
         else:
-            log.warning('Called RealTimeManager.create_realtime, but job is None')
+            log.warning('Called RealTimeManager.create_realtime, but job (%s) or tool (%s) is None', job, tool)
 
     def get_nonterminal_for_user_by_trans(self, trans):
         if trans.user:
@@ -210,16 +217,15 @@ class RealTimeManager(object):
                     query = query.filter(or_(*t))
             return query
         jobs = build_and_apply_filters(jobs, trans.app.model.Job.non_ready_states, lambda s: trans.app.model.Job.state == s)
-        rtts = trans.sa_session.query(trans.app.model.RealTimeTool).filter(trans.app.model.RealTimeTool.job_id.in_([job.id for job in jobs]))
-        return trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).filter(trans.app.model.RealTimeToolEntryPoint.realtime_id.in_([rtt.id for rtt in rtts]))
+        return trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).filter(trans.app.model.RealTimeToolEntryPoint.job_id.in_([job.id for job in jobs]))
 
-    def can_access_realtime(self, trans, realtime):
-        if realtime:
+    def can_access_job(self, trans, job):
+        if job:
             if trans.user is None:
                 galaxy_session = trans.get_galaxy_session()
-                if galaxy_session is None or realtime.session_id != galaxy_session.id:
+                if galaxy_session is None or job.session_id != galaxy_session.id:
                     return False
-            elif realtime.user != trans.user:
+            elif job.user != trans.user:
                 return False
         else:
             return False
@@ -227,10 +233,16 @@ class RealTimeManager(object):
 
     def can_access_entry_point(self, trans, entry_point):
         if entry_point:
-            return self.can_access_realtime(trans, entry_point.realtime)
+            return self.can_access_job(trans, entry_point.job)
         return False
 
-    def stop(self, trans, realtime):
+    def can_access_entry_points(self, trans, entry_points):
+        for ep in entry_points:
+            if not self.can_access_entry_point(trans, ep):
+                return False
+        return True
+
+    def stop0(self, trans, realtime):
         try:
             self.remove_realtime(realtime)
             job = realtime.job
@@ -247,8 +259,18 @@ class RealTimeManager(object):
             return False
         return True
 
-    def remove_realtime(self, realtime):
+    def remove_realtime0(self, realtime):
         return self.propagator.remove_realtime(realtime)
 
-    def remove_entry_point(self, entry_point):
-        return self.propagator.remove_entry_point(entry_point)
+    def remove_entry_points(self, entry_points):
+        if entry_points:
+            for entry_point in entry_points:
+                self.remove_entry_point(entry_point, flush=False)
+            self.sa_session.flush()
+
+    def remove_entry_point(self, entry_point, flush=True):
+        entry_point.deleted = True
+        self.sa_session.add(entry_point)
+        if flush:
+            self.sa_session.flush()
+        self.propagator.remove_entry_point(entry_point)
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 3d8721f214a..328ea4b3df7 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -1327,36 +1327,22 @@ class JobContainerAssociation(RepresentById):
         self.container_info = container_info or {}
 
 
-class RealTimeTool(RepresentById):
-    def __init__(self, job=None, user=None, session=None, dataset=None):
-        self.job = job
-        self.user = user
-        self.galaxy_session = session
-        self.dataset = dataset
-
-    @property
-    def active(self):
-        # FIXME: don't included queued?
-        return not self.job.finished
-
-    def to_dict(self, *args, **kwds):
-        return dict(job_id=self.job.id, user_id=self.user.id, galaxy_session_id=self.galaxy_session.id, dataset_id=self.dataset.id, active=self.active)
-
-
 class RealTimeToolEntryPoint(RepresentById):
-    def __init__(self, realtime=None, name=None, token=None, tool_port=None, entry_url=None,
-                 host=None, port=None, protocol=None, configured=False):
-        self.realtime = realtime
+    def __init__(self, job=None, name=None, token=None, tool_port=None, host=None, port=None, protocol=None,
+                 entry_url=None, info=None, configured=False, deleted=False):
+        self.job = job
         self.name = name
         if not token:
             token = uuid4().get_hex()
         self.token = token
         self.tool_port = tool_port
-        self.entry_url = entry_url
         self.host = host
         self.port = port
         self.protocol = protocol
+        self.entry_url = entry_url
+        self.info = info or {}
         self.configured = configured
+        self.deleted = deleted
 
     def to_dict(self, *args, **kwds):
         rval = dict(realtime_id=self.realtime.id)
@@ -1366,8 +1352,9 @@ class RealTimeToolEntryPoint(RepresentById):
 
     @property
     def active(self):
-        if self.configured:
-            return self.realtime.active
+        if self.configured and not self.deleted:
+            # FIXME: don't included queued?
+            return not self.job.finished
         return False
 
 
diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py
index 1b8548eaf34..e72984702b3 100644
--- a/lib/galaxy/model/mapping.py
+++ b/lib/galaxy/model/mapping.py
@@ -740,28 +740,20 @@ model.GenomeIndexToolData.table = Table(
     Column("indexer", String(64)),
     Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True))
 
-model.RealTimeTool.table = Table(
-    "realtime_tool", metadata,
+model.RealTimeToolEntryPoint.table = Table(
+    "realtimetool_entry_point", metadata,
     Column("id", Integer, primary_key=True),
     Column("job_id", Integer, ForeignKey("job.id"), index=True),
-    Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True),
-    Column("session_id", Integer, ForeignKey("galaxy_session.id"), index=True),
-    Column("dataset_id", Integer, ForeignKey("history_dataset_association.id"), index=True),
-    Column("created_time", DateTime, default=now),
-    Column("modified_time", DateTime, default=now, onupdate=now))
-
-model.RealTimeToolEntryPoint.table = Table(
-    "realtime_tool_entry_point", metadata,
-    Column("id", Integer, primary_key=True),
-    Column("realtime_id", Integer, ForeignKey("realtime_tool.id"), index=True),
     Column("name", TEXT),
     Column("token", TEXT),
     Column("tool_port", Integer),
-    Column("entry_url", TEXT),
     Column("host", TEXT),
     Column("port", Integer),
     Column("protocol", TEXT),
+    Column("entry_url", TEXT),
+    Column("info", JSONType, nullable=True),
     Column("configured", Boolean, default=False),
+    Column("deleted", Boolean, default=False),
     Column("created_time", DateTime, default=now),
     Column("modified_time", DateTime, default=now, onupdate=now))
 
@@ -2125,15 +2117,8 @@ mapper(model.GenomeIndexToolData, model.GenomeIndexToolData.table, properties=di
     transfer=relation(model.TransferJob, backref='transfer_job')
 ))
 
-mapper(model.RealTimeTool, model.RealTimeTool.table, properties=dict(
-    job=relation(model.Job, backref=backref('realtime_tool', uselist=False), uselist=False),
-    dataset=relation(model.HistoryDatasetAssociation, backref=backref('realtime_tool', uselist=False), uselist=False),
-    user=relation(model.User),
-    galaxy_session=relation(model.GalaxySession)
-))
-
 mapper(model.RealTimeToolEntryPoint, model.RealTimeToolEntryPoint.table, properties=dict(
-    realtime=relation(model.RealTimeTool, backref='entry_points', uselist=False)
+    job=relation(model.Job, backref=backref('realtimetool_entry_points', uselist=True), uselist=False)
 ))
 
 mapper(model.JobContainerAssociation, model.JobContainerAssociation.table, properties=dict(
diff --git a/lib/galaxy/model/migrate/versions/0149_add_realtimetools.py b/lib/galaxy/model/migrate/versions/0149_add_realtimetools.py
index 5e2764b9a27..c60b2b5defc 100644
--- a/lib/galaxy/model/migrate/versions/0149_add_realtimetools.py
+++ b/lib/galaxy/model/migrate/versions/0149_add_realtimetools.py
@@ -13,29 +13,20 @@ from galaxy.model.orm.now import now
 log = logging.getLogger(__name__)
 metadata = MetaData()
 
-realtime_tool = Table(
-    "realtime_tool", metadata,
+realtimetool_entry_point = Table(
+    "realtimetool_entry_point", metadata,
     Column("id", Integer, primary_key=True),
     Column("job_id", Integer, ForeignKey("job.id"), index=True),
-    Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True),
-    Column("session_id", Integer, ForeignKey("galaxy_session.id"), index=True),
-    Column("dataset_id", Integer, ForeignKey("history_dataset_association.id"), index=True),
-    Column("created_time", DateTime, default=now),
-    Column("modified_time", DateTime, default=now, onupdate=now))
-
-
-realtime_tool_entry_point = Table(
-    "realtime_tool_entry_point", metadata,
-    Column("id", Integer, primary_key=True),
-    Column("realtime_id", Integer, ForeignKey("realtime_tool.id"), index=True),
     Column("name", TEXT),
     Column("token", TEXT),
     Column("tool_port", Integer),
     Column("host", TEXT),
     Column("port", Integer),
     Column("protocol", TEXT),
-    Column("configured", Boolean, default=False),
     Column("entry_url", TEXT),
+    Column("info", JSONType, nullable=True),
+    Column("configured", Boolean, default=False),
+    Column("deleted", Boolean, default=False),
     Column("created_time", DateTime, default=now),
     Column("modified_time", DateTime, default=now, onupdate=now))
 
@@ -55,37 +46,27 @@ def upgrade(migrate_engine):
     metadata.bind = migrate_engine
     metadata.reflect()
 
-    try:
-        realtime_tool.create()
-    except Exception:
-        log.exception("Failed to create realtime_tool table")
-
     try:
         job_container_association.create()
     except Exception:
         log.exception("Failed to create job_container_association table")
 
     try:
-        realtime_tool_entry_point.create()
+        realtimetool_entry_point.create()
     except Exception:
-        log.exception("Failed to create realtime_tool_entry_point table")
+        log.exception("Failed to create realtimetool_entry_point table")
 
 
 def downgrade(migrate_engine):
     metadata.bind = migrate_engine
     metadata.reflect()
 
-    try:
-        realtime_tool.drop()
-    except Exception:
-        log.exception("Failed to drop realtime_tool table")
-
     try:
         job_container_association.drop()
     except Exception:
         log.exception("Failed to drop job_container_association table")
 
     try:
-        realtime_tool_entry_point.drop()
+        realtimetool_entry_point.drop()
     except Exception:
-        log.exception("Failed to drop realtime_tool_entry_point table")
+        log.exception("Failed to drop realtimetool_entry_point table")
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 85dc9fe3d1f..6bb443be304 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -2325,10 +2325,9 @@ class RealTimeTool(Tool):
 
     def __remove_realtime_by_job(self, job):
         if job:
-            rtt = job.realtime_tool
-            log.debug('__remove_realtime_by_job: %s', rtt)
-            if rtt:
-                self.app.realtime_manager.remove_realtime(rtt)
+            eps = job.realtimetool_entry_points
+            log.debug('__remove_realtime_by_job: %s', eps)
+            self.app.realtime_manager.remove_entry_points(eps)
         else:
             log.warning("Could not determine job to stop RealTimeTool: %s", job)
 
@@ -2344,9 +2343,9 @@ class RealTimeTool(Tool):
 
     def get_view_result(self, job=None):
         if job:
-            realtimetool = job.realtime_tool
-            if realtimetool:
-                return self.app.url_for(controller='realtime', action='index', realtime_id=self.app.security.encode_id(realtimetool.id))
+            eps = job.realtimetool_entry_points
+            if eps:
+                return self.app.url_for(controller='realtime', action='index', entry_point_id=[self.app.security.encode_id(ep.id) for ep in eps])
         return None
 
 
diff --git a/lib/galaxy/tools/actions/realtime.py b/lib/galaxy/tools/actions/realtime.py
index 01af6dd3de6..b40c717b993 100644
--- a/lib/galaxy/tools/actions/realtime.py
+++ b/lib/galaxy/tools/actions/realtime.py
@@ -11,5 +11,5 @@ class RealTimeToolAction(DefaultToolAction):
     def execute(self, tool, trans, **kwds):
         rval = super(RealTimeToolAction, self).execute(tool, trans, **kwds)
         job, output_dict = rval
-        trans.app.realtime_manager.create_realtime(trans, job, tool)
+        trans.app.realtime_manager.create_realtime(job, tool)
         return rval
diff --git a/lib/galaxy/tools/deps/docker_util.py b/lib/galaxy/tools/deps/docker_util.py
index 162eb3a7584..9f53e4bda64 100644
--- a/lib/galaxy/tools/deps/docker_util.py
+++ b/lib/galaxy/tools/deps/docker_util.py
@@ -222,3 +222,17 @@ def _docker_prefix(
     if host:
         command_parts.extend(["-H", host])
     return command_parts
+
+
+def parse_port_text(port_text):
+    ports = None
+    if port_text is not None:
+        ports = {}
+        for line in port_text.strip().split('\n'):
+            tool, host = line.split(" -> ", 1)
+            hostname, port = host.split(':')
+            port = int(port)
+            tool_p, tool_prot = tool.split("/")
+            tool_p = int(tool_p)
+            ports[tool_p] = dict(tool_port=tool_p, host=hostname, port=port, protocol=tool_prot)
+    return ports
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index 72ef2717e48..77168273dde 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -58,18 +58,22 @@ class RealTime(BaseUIController):
     _rtt_ep_grid = RealTimeToolEntryPointListGrid()
 
     @web.expose
-    def index(self, trans, realtime_id=None, **kwd):
-        realtime = None
-        if realtime_id:
-            realtime_id = self.decode_id(realtime_id)
-            realtime = trans.sa_session.query(trans.app.model.RealTimeTool).get(realtime_id)
-        if realtime:
-            if trans.app.realtime_manager.can_access_realtime(trans, realtime):
-                if realtime.entry_points:
-                    if len(realtime.entry_points) == 1:
-                        return self.access_entry_point(trans, entry_point_id=trans.security.encode_id(realtime.entry_points[0].id))
-                    else:
-                        return trans.response.send_redirect(url_for(controller="realtime", action="list"))
+    def index(self, trans, entry_point_id=None, job_id=None, **kwd):
+        eps = []
+        if entry_point_id:
+            if not isinstance(entry_point_id, list):
+                entry_point_id = [entry_point_id]
+            eps = [trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).get(self.decode_id(ep_id)) for ep_id in entry_point_id]
+        if job_id:
+            job = trans.sa_session.query(trans.app.model.Job).get(self.decode_id(job_id))
+            for ep in job.realtimetool_entry_points:
+                eps.append(ep)
+        if eps:
+            if trans.app.realtime_manager.can_access_entry_points(trans, eps):
+                if len(eps) == 1:
+                    return self.access_entry_point(trans, entry_point_id=trans.security.encode_id(eps[0].id))
+                else:
+                    return trans.response.send_redirect(url_for(controller="realtime", action="list"))
             else:
                 return trans.show_error_message('Access not authorized.')
         return trans.show_error_message('RealTimeTool instance not found')
@@ -84,12 +88,14 @@ class RealTime(BaseUIController):
             entry_point_id = self.decode_id(entry_point_id)
             entry_point = trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).get(entry_point_id)
             if trans.app.realtime_manager.can_access_entry_point(trans, entry_point):
-                if entry_point.realtime.active and entry_point.configured:
+                if entry_point.active:
                     rval = '%s//%s.%s.%s.%s.%s/' % (trans.request.host_url.split('//', 1)[0], entry_point.__class__.__name__.lower(), trans.security.encode_id(entry_point.id),
                             entry_point.token, trans.app.config.realtime_prefix, trans.request.host)
                     if entry_point.entry_url:
                         rval = '%s%s' % (rval, entry_point.entry_url)
                     return trans.response.send_redirect(rval)
+                elif entry_point.deleted:
+                    return trans.show_error_message('RealTimeTool has ended. You will have to start a new one.')
                 else:
                     return trans.show_error_message('RealTimeTool is not active. If you recently launched this tool it may not be ready yet, please wait a moment and refresh this page.')
         return trans.show_error_message('Access not authorized.')
@@ -102,7 +108,6 @@ class RealTime(BaseUIController):
         message = None
         if operation:
             eps = []
-            rtts = []
             ids = kwargs.get('id', None)
             if ids:
                 if not isinstance(ids, list):
@@ -112,18 +117,21 @@ class RealTime(BaseUIController):
                     entry_point = trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).get(entry_point_id)
                     if trans.app.realtime_manager.can_access_entry_point(trans, entry_point):
                         eps.append(entry_point)
-                        if entry_point.realtime not in rtts:
-                            rtts.append(entry_point.realtime)
-            if rtts:
+            if eps:
                 failed = []
                 succeeded = []
+                jobs = []
                 if operation == 'stop':
-                    for realtime in rtts:
-                        stopped = trans.app.realtime_manager.stop(trans, realtime)
-                        if stopped:
-                            succeeded.append(realtime)
+                    for ep in eps:
+                        if ep.job not in jobs:
+                            stopped = trans.app.realtime_manager.stop(trans, ep)
+                            if stopped:
+                                succeeded.append(ep)
+                                jobs.append(ep.job)
+                            else:
+                                failed.append(ep)
                         else:
-                            failed.append(realtime)
+                            succeeded.append(ep)
                     if failed:
                         message = 'Unable to stop %i RealTimeTools.' % (len(failed))
                     if succeeded:
@@ -131,7 +139,7 @@ class RealTime(BaseUIController):
         return self._rtt_ep_grid(trans, message=message)
 
     @web.expose_api
-    def stop(self, trans, realtime_id=None, realtime_entry_id=None):
+    def stop0(self, trans, realtime_id=None, realtime_entry_id=None):
         """List all available realtimetools"""
         if realtime_id:
             realtime = trans.app.model.RealTimeTool.get(trans.security.decode_id(realtime_id))

From 27fe34f1f5d044c613f48c7d22ca6c9c0d88bdf9 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Thu, 21 Mar 2019 09:00:43 -0400
Subject: [PATCH 014/114] remove debug statement

---
 lib/galaxy/managers/realtime.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index 44acf51a246..ed1cc8d3a0f 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -165,7 +165,6 @@ class RealTimeManager(object):
         # There can be multiple entry points that reference the same tool port (could have different entry URLs)
         configured = []
         not_configured = []
-        log.debug('ports_dict %s', str(ports_dict))
         for ep in job.realtimetool_entry_points:
             port_dict = ports_dict.get(ep.tool_port, None)
             if port_dict is None:

From e037148fc87787b9fde192cd4c8365d058a34538 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Thu, 21 Mar 2019 10:53:17 -0400
Subject: [PATCH 015/114] comma

---
 lib/galaxy/tools/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 142db211cda..e5e8ac4b93f 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -3072,7 +3072,7 @@ class FilterFromFileTool(DatabaseOperationTool):
 
 # Populate tool_type to ToolClass mappings
 tool_types = {}
-for tool_class in [Tool, SetMetadataTool, OutputParameterJSONTool, ExpressionTool, RealTimeTool
+for tool_class in [Tool, SetMetadataTool, OutputParameterJSONTool, ExpressionTool, RealTimeTool,
                    DataManagerTool, DataSourceTool, AsyncDataSourceTool,
                    UnzipCollectionTool, ZipCollectionTool, MergeCollectionTool, RelabelFromFileTool, FilterFromFileTool,
                    BuildListCollectionTool, ExtractDatasetCollectionTool,

From 91f5f6ffe915da607c5494afdde4291e1eb1eb2c Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Thu, 21 Mar 2019 17:06:56 -0400
Subject: [PATCH 016/114] fix for displaying realtimetool grid on refresh.

---
 .../scripts/entry/analysis/AnalysisRouter.js  |  2 +-
 client/galaxy/scripts/layout/menu.js          |  2 +-
 lib/galaxy/managers/realtime.py               | 11 ++--
 lib/galaxy/webapps/galaxy/buildapp.py         |  1 +
 .../webapps/galaxy/controllers/realtime.py    | 53 ++++++++++---------
 5 files changed, 35 insertions(+), 34 deletions(-)

diff --git a/client/galaxy/scripts/entry/analysis/AnalysisRouter.js b/client/galaxy/scripts/entry/analysis/AnalysisRouter.js
index 634be42c9b8..31970fdd1f1 100644
--- a/client/galaxy/scripts/entry/analysis/AnalysisRouter.js
+++ b/client/galaxy/scripts/entry/analysis/AnalysisRouter.js
@@ -73,7 +73,7 @@ export const getAnalysisRouter = Galaxy =>
             "(/)custom_builds": "show_custom_builds",
             "(/)datasets/edit": "show_dataset_edit_attributes",
             "(/)datasets/error": "show_dataset_error",
-            "(/)realtime/list": "show_realtime_list"
+            "(/)realtime_entry_points(/)list": "show_realtime_list"
         },
 
         require_login: ["show_user", "show_user_form", "show_workflows"],
diff --git a/client/galaxy/scripts/layout/menu.js b/client/galaxy/scripts/layout/menu.js
index a9f225922a3..312272f575d 100644
--- a/client/galaxy/scripts/layout/menu.js
+++ b/client/galaxy/scripts/layout/menu.js
@@ -301,7 +301,7 @@ var Collection = Backbone.Collection.extend({
                     },
                     {
                         title: _l("Active RealTimeTools"),
-                        url: "realtime/list",
+                        url: "realtime_entry_points/list",
                         target: "__use_router__"
                     }
                 ]
diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index ed1cc8d3a0f..34d451f6822 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -241,20 +241,19 @@ class RealTimeManager(object):
                 return False
         return True
 
-    def stop0(self, trans, realtime):
+    def stop(self, trans, entry_point):
         try:
-            self.remove_realtime(realtime)
-            job = realtime.job
+            self.remove_entry_point(entry_point)
+            job = entry_point.job
             if not job.finished:
-                # FIXME: Need stop container, but set to job OK.
-                log.debug('Stopping Job: %s for RealTimeTool: %s', job, realtime)
+                log.debug('Stopping Job: %s for RealTimeToolEntryPoint: %s', job, entry_point)
                 job.mark_deleted(trans.app.config.track_jobs_in_database)
                 # This self.job_manager.stop(job) does nothing without changing job.state, manually or e.g. with .mark_deleted()
                 self.job_manager.stop(job)
                 trans.sa_session.add(job)
                 trans.sa_session.flush()
         except Exception as e:
-            log.debug('Unable to stop job for RealTimeTool: %s', e)
+            log.debug('Unable to stop job for RealTimeToolEntryPoint (%s): %s', entry_point, e)
             return False
         return True
 
diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py
index 87a166cd164..d5620c10871 100644
--- a/lib/galaxy/webapps/galaxy/buildapp.py
+++ b/lib/galaxy/webapps/galaxy/buildapp.py
@@ -147,6 +147,7 @@ def app_factory(global_conf, load_app_kwds={}, **kwargs):
     webapp.add_client_route('/workflows/run')
     webapp.add_client_route('/workflows/import')
     webapp.add_client_route('/custom_builds')
+    webapp.add_client_route('/realtime_entry_points/list')
 
     # ==== Done
     # Indicate that all configuration settings have been provided
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index 77168273dde..06d3bc318a4 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -19,12 +19,24 @@ from galaxy.web.framework.helpers import (
 log = logging.getLogger(__name__)
 
 
+class JobStatusColumn(grids.StateColumn):
+    def get_value(self, trans, grid, item):
+        return super(JobStatusColumn, self).get_value(trans, grid, item.job)
+
+
 class RealTimeToolEntryPointListGrid(grids.Grid):
     def get_url_args(item):
         """
         Returns dictionary used to create item link.
         """
-        url_kwargs = dict(controller="realtime", action="access_entry_point_ri", entry_point_id=item.id)
+        url_kwargs = dict(controller="realtime", action="access_entry_point", id=item.id)
+        return url_kwargs
+
+    def get_url_args_job(item):
+        """
+        Returns dictionary used to create item link.
+        """
+        url_kwargs = dict(controller="api/jobs", action='show', id=item.job.id)
         return url_kwargs
 
     use_panels = True
@@ -35,6 +47,7 @@ class RealTimeToolEntryPointListGrid(grids.Grid):
     columns = [
         grids.TextColumn("Name", key="name", filterable="advanced", link=get_url_args),
         grids.GridColumn("Active", key="active"),
+        JobStatusColumn("Job Info", key="job_state", model_class=model.Job),
         grids.GridColumn("Created", key="created_time", format=time_ago),
         grids.GridColumn("Last Updated", key="modified_time", format=time_ago),
     ]
@@ -55,7 +68,7 @@ class RealTimeToolEntryPointListGrid(grids.Grid):
 
 
 class RealTime(BaseUIController):
-    _rtt_ep_grid = RealTimeToolEntryPointListGrid()
+    entry_point_grid = RealTimeToolEntryPointListGrid()
 
     @web.expose
     def index(self, trans, entry_point_id=None, job_id=None, **kwd):
@@ -71,7 +84,7 @@ class RealTime(BaseUIController):
         if eps:
             if trans.app.realtime_manager.can_access_entry_points(trans, eps):
                 if len(eps) == 1:
-                    return self.access_entry_point(trans, entry_point_id=trans.security.encode_id(eps[0].id))
+                    return self.access_entry_point(trans, id=trans.security.encode_id(eps[0].id))
                 else:
                     return trans.response.send_redirect(url_for(controller="realtime", action="list"))
             else:
@@ -79,13 +92,10 @@ class RealTime(BaseUIController):
         return trans.show_error_message('RealTimeTool instance not found')
 
     @web.expose
-    def access_entry_point_ri(self, trans, entry_point_id=None):
-        return trans.response.send_redirect(url_for(controller='realtime', action='access_entry_point', entry_point_id=trans.security.encode_id(entry_point_id)))
-
-    @web.expose
-    def access_entry_point(self, trans, entry_point_id=None):
-        if entry_point_id:
-            entry_point_id = self.decode_id(entry_point_id)
+    def access_entry_point(self, trans, id=None):
+        # Because of auto id encoding needed for link from grid, the item.id keyword must be 'id'
+        if id:
+            entry_point_id = self.decode_id(id)
             entry_point = trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).get(entry_point_id)
             if trans.app.realtime_manager.can_access_entry_point(trans, entry_point):
                 if entry_point.active:
@@ -103,9 +113,9 @@ class RealTime(BaseUIController):
     @web.expose_api
     def list(self, trans, **kwargs):
         """List all available realtimetools"""
-        log.debug('list kargs %s', str(kwargs))
         operation = kwargs.get('operation', None)
         message = None
+        status = None
         if operation:
             eps = []
             ids = kwargs.get('id', None)
@@ -134,20 +144,11 @@ class RealTime(BaseUIController):
                             succeeded.append(ep)
                     if failed:
                         message = 'Unable to stop %i RealTimeTools.' % (len(failed))
+                        status = 'error'
                     if succeeded:
                         message = 'Stopped %i RealTimeTools.' % (len(succeeded))
-        return self._rtt_ep_grid(trans, message=message)
-
-    @web.expose_api
-    def stop0(self, trans, realtime_id=None, realtime_entry_id=None):
-        """List all available realtimetools"""
-        if realtime_id:
-            realtime = trans.app.model.RealTimeTool.get(trans.security.decode_id(realtime_id))
-        elif realtime_entry_id:
-            realtime = trans.app.model.RealTimeToolEntryPoint.get(trans.security.decode_id(realtime_entry_id)).realtime
-        else:
-            return dict(message="Can't stop: RealTimeTool not provided.", state='error')
-        stopped = trans.app.realtime_manager.stop(trans, realtime)
-        if stopped:
-            return dict(message="RealTimeTool has been stopped", state='ok')
-        return dict(message="Unable to stop RealTimeTool", state='error')
+                        status = 'ok'
+        if message and status:
+            kwargs['message'] = message
+            kwargs['status'] = status
+        return self.entry_point_grid(trans, **kwargs)

From e8da30814d6982d5c5cfd6a6adfc4877e40632ba Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 22 Mar 2019 09:14:16 -0400
Subject: [PATCH 017/114] Ports for yaml tools.

---
 lib/galaxy/jobs/runners/__init__.py           |  4 +--
 lib/galaxy/managers/realtime.py               |  2 +-
 lib/galaxy/tools/__init__.py                  |  2 +-
 lib/galaxy/tools/parser/interface.py          |  2 +-
 lib/galaxy/tools/parser/xml.py                | 30 ++++++++++---------
 lib/galaxy/tools/parser/yaml.py               |  3 ++
 .../webapps/galaxy/controllers/realtime.py    |  2 +-
 7 files changed, 25 insertions(+), 20 deletions(-)

diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py
index bd8207058bd..e1cddcbae1e 100644
--- a/lib/galaxy/jobs/runners/__init__.py
+++ b/lib/galaxy/jobs/runners/__init__.py
@@ -417,8 +417,8 @@ class BaseJobRunner(object):
             compute_tmp_directory = job_wrapper.tmp_directory()
 
         tool = job_wrapper.tool
-        # TODO: template tool ports?
-        guest_ports = [port.get('port') for port in tool.tool_ports]
+        # TODO: template RealTimeTool ports?
+        guest_ports = [port.get('port') for port in tool.ports]
         tool_info = ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through, guest_ports=guest_ports)
         job_info = JobInfo(
             compute_working_directory,
diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index 34d451f6822..1f89b8c4d8c 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -152,7 +152,7 @@ class RealTimeManager(object):
         self.propagator = RealtimeSqlite(app.config.proxy_session_map, app.security.encode_id)
 
     def create_entry_points(self, job, tool, flush=True):
-        for entry in tool.tool_ports:
+        for entry in tool.ports:
             ep = self.model.RealTimeToolEntryPoint(job=job, tool_port=entry['port'], entry_url=entry['url'], name=entry['name'])
             self.sa_session.add(ep)
         if flush:
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index e5e8ac4b93f..618dc4f0bf6 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -807,7 +807,7 @@ class Tool(Dictifiable):
         self.__parse_trackster_conf(tool_source)
         # Record macro paths so we can reload a tool if any of its macro has changes
         self._macro_paths = tool_source.macro_paths()
-        self.tool_ports = tool_source.parse_tool_ports()
+        self.ports = tool_source.parse_ports()
 
     def __parse_legacy_features(self, tool_source):
         self.code_namespace = dict()
diff --git a/lib/galaxy/tools/parser/interface.py b/lib/galaxy/tools/parser/interface.py
index 935aaf259ae..7cfd0d5ee99 100644
--- a/lib/galaxy/tools/parser/interface.py
+++ b/lib/galaxy/tools/parser/interface.py
@@ -123,7 +123,7 @@ class ToolSource(object):
         """
 
     @abstractmethod
-    def parse_tool_ports(self):
+    def parse_ports(self):
         """ Return port templates to expose.
         """
 
diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py
index 2fb41a303d4..38215edb0ba 100644
--- a/lib/galaxy/tools/parser/xml.py
+++ b/lib/galaxy/tools/parser/xml.py
@@ -193,26 +193,28 @@ class XmlToolSource(ToolSource):
             return ParallelismInfo(parallelism)
         return parallelism_info
 
+    def parse_ports(self):
+        ports_el = self.root.find("ports")
+        ports = []
+        if ports_el is None:
+            return ports
+        for port_el in ports_el.findall("port"):
+            port = port_el.text.strip()
+            url = port_el.find("url")
+            if url is not None:
+                url = url.text.strip()
+            name = port_el.get('name', None)
+            if name:
+                name = name.strip()
+            ports.append(dict(port=port, url=url, name=name))
+        return ports
+
     def parse_hidden(self):
         hidden = xml_text(self.root, "hidden")
         if hidden:
             hidden = string_as_bool(hidden)
         return hidden
 
-    def parse_tool_ports(self):
-        # TODO: allow ports to be templated, for now, we will just use static value
-        tool_ports_el = self.root.find("tool_ports")
-        tool_ports = []
-        if tool_ports_el is None:
-            return tool_ports
-        for tool_port_el in tool_ports_el.findall("port"):
-            port = tool_port_el.text.strip()
-            entry_path = tool_port_el.find("url")
-            if entry_path is not None:
-                entry_path = entry_path.text.strip()
-            tool_ports.append(dict(port=port, url=entry_path, name=tool_port_el.get('name', None)))
-        return tool_ports
-
     def parse_redirect_url_params_elem(self):
         return self.root.find("redirect_url_params")
 
diff --git a/lib/galaxy/tools/parser/yaml.py b/lib/galaxy/tools/parser/yaml.py
index 52ae9d4cb07..a8dc0ec4f22 100644
--- a/lib/galaxy/tools/parser/yaml.py
+++ b/lib/galaxy/tools/parser/yaml.py
@@ -179,6 +179,9 @@ class YamlToolSource(ToolSource):
     def parse_profile(self):
         return self.root_dict.get("profile", "16.04")
 
+    def parse_ports(self):
+        return self.root_dict.get("ports", [])
+
 
 def _parse_test(i, test_dict):
     inputs = test_dict["inputs"]
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index 06d3bc318a4..895c04bda54 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -110,7 +110,7 @@ class RealTime(BaseUIController):
                     return trans.show_error_message('RealTimeTool is not active. If you recently launched this tool it may not be ready yet, please wait a moment and refresh this page.')
         return trans.show_error_message('Access not authorized.')
 
-    @web.expose_api
+    @web.expose_api_anonymous
     def list(self, trans, **kwargs):
         """List all available realtimetools"""
         operation = kwargs.get('operation', None)

From 464aa35e4fc7721b722ddf02982f74880ff468d9 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 22 Mar 2019 13:15:38 -0400
Subject: [PATCH 018/114] update tools xsd

---
 lib/galaxy/tools/xsd/galaxy.xsd | 52 +++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd
index 0f59437410f..20365f13c8f 100644
--- a/lib/galaxy/tools/xsd/galaxy.xsd
+++ b/lib/galaxy/tools/xsd/galaxy.xsd
@@ -48,6 +48,7 @@ A ``data_source`` tool contains a few more relevant attributes.
         <xs:element name="edam_topics" type="EdamTopics" minOccurs="0"/>
         <xs:element name="edam_operations" type="EdamOperations" minOccurs="0"/>
         <xs:element name="requirements" type="Requirements" minOccurs="0"/>
+        <xs:element name="ports" type="Ports" minOccurs="0" maxOccurs="1" />
         <xs:element name="description" type="xs:string" minOccurs="0">
           <xs:annotation gxdocs:best_practices="tool-descriptions">
             <xs:documentation xml:lang="en"><![CDATA[The value is displayed in
@@ -215,6 +216,56 @@ For more information, see https://planemo.readthedocs.io/en/latest/writing_advan
     </xs:sequence>
   </xs:complexType>
 
+  <xs:complexType name="Ports">
+    <xs:annotation>
+      <xs:documentation xml:lang="en"><![CDATA[
+
+This is a container tag set for the ``port`` and ``url`` tags
+described in greater detail below. ``ports``s describe RealTimeTool entry points
+to a tool.
+
+]]></xs:documentation>
+    </xs:annotation>
+    <xs:sequence>
+      <xs:element name="port" type="Port" minOccurs="0" maxOccurs="unbounded"/>
+    </xs:sequence>
+  </xs:complexType>
+
+  <xs:complexType name="Port" mixed="true">
+    <xs:annotation>
+      <xs:documentation xml:lang="en"><![CDATA[
+
+This tag set is contained within the ``<ports>`` tag set. Access ports and
+entry point urls are included in this tag set. These are used by RealTimeTools
+to provide access to graphical tools in real time.
+
+```xml
+<ports>
+    <port name="Example name">80<url>landing/index.html</url></port>
+</ports>
+```
+
+]]></xs:documentation>
+    </xs:annotation>
+    <xs:sequence>
+      <xs:element name="url" type="PortURL" minOccurs="0" maxOccurs="1"/>
+    </xs:sequence
+        <xs:attribute name="name" type="xs:string" use="required">
+          <xs:annotation>
+            <xs:documentation xml:lang="en">This value defines the name of the entry point.</xs:documentation>
+          </xs:annotation>
+        </xs:attribute>
+  </xs:complexType>
+  <xs:complexType name="PortURL">
+    <xs:annotation>
+      <xs:documentation xml:lang="en"><![CDATA[
+
+This tag set is contained within the ``<port>`` tag set. It contains the entry URL.
+
+]]></xs:documentation>
+    </xs:annotation>
+  </xs:complexType>
+
   <xs:complexType name="ToolAction">
     <xs:annotation>
       <xs:documentation xml:lang="en">Describe the backend Python action to execute for this Galaxy tool.</xs:documentation>
@@ -5337,6 +5388,7 @@ and ``bibtex`` are the only supported options.</xs:documentation>
     <xs:restriction base="xs:string">
       <xs:enumeration value="data_source"/>
       <xs:enumeration value="manage_data"/>
+      <xs:enumeration value="realtime"/>
       <xs:enumeration value="expression"/>
     </xs:restriction>
   </xs:simpleType>

From 523f648e36426be421d7b654aba993d7faf39ead Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 22 Mar 2019 13:46:12 -0400
Subject: [PATCH 019/114] over zealous delete key

---
 lib/galaxy/tools/xsd/galaxy.xsd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd
index 20365f13c8f..b9628491a00 100644
--- a/lib/galaxy/tools/xsd/galaxy.xsd
+++ b/lib/galaxy/tools/xsd/galaxy.xsd
@@ -249,7 +249,7 @@ to provide access to graphical tools in real time.
     </xs:annotation>
     <xs:sequence>
       <xs:element name="url" type="PortURL" minOccurs="0" maxOccurs="1"/>
-    </xs:sequence
+    </xs:sequence>
         <xs:attribute name="name" type="xs:string" use="required">
           <xs:annotation>
             <xs:documentation xml:lang="en">This value defines the name of the entry point.</xs:documentation>

From d64ba78b757ac161ebc4fb80d2f5824bf493a830 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 22 Mar 2019 14:06:19 -0400
Subject: [PATCH 020/114] flake

---
 lib/galaxy/tools/__init__.py                          | 2 +-
 lib/galaxy/web/realtime/rpc/key_type_token_mapping.py | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 618dc4f0bf6..47800835559 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -34,10 +34,10 @@ from galaxy.tools.actions import DefaultToolAction
 from galaxy.tools.actions.data_manager import DataManagerToolAction
 from galaxy.tools.actions.data_source import DataSourceToolAction
 from galaxy.tools.actions.model_operations import ModelOperationToolAction
+from galaxy.tools.actions.realtime import RealTimeToolAction
 from galaxy.tools.deps import (
     CachedDependencyManager,
 )
-from galaxy.tools.actions.realtime import RealTimeToolAction
 from galaxy.tools.fetcher import ToolLocationFetcher
 from galaxy.tools.parameters import (
     check_param,
diff --git a/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py b/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
index 4471b38638e..a25cc6e0b56 100644
--- a/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
+++ b/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
@@ -1,6 +1,5 @@
 import itertools
 import sqlite3
-
 from threading import RLock
 from time import time
 

From c96011322fbce3a30791afa2be76b0ab32e021e9 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 22 Mar 2019 14:12:27 -0400
Subject: [PATCH 021/114] flake

---
 lib/galaxy/tools/deps/container_classes.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/galaxy/tools/deps/container_classes.py b/lib/galaxy/tools/deps/container_classes.py
index 3160968c649..6594f92b564 100644
--- a/lib/galaxy/tools/deps/container_classes.py
+++ b/lib/galaxy/tools/deps/container_classes.py
@@ -4,9 +4,8 @@ from abc import (
     ABCMeta,
     abstractmethod
 )
-from uuid import uuid4
-
 from logging import getLogger
+from uuid import uuid4
 
 import six
 

From cf314d2d1d50140a5caf1c435bd6677c263ae1f2 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 22 Mar 2019 15:14:36 -0400
Subject: [PATCH 022/114] naming update

---
 config/galaxy.yml.realtime                            | 4 ++--
 lib/galaxy/config.py                                  | 2 ++
 lib/galaxy/managers/realtime.py                       | 2 +-
 lib/galaxy/web/realtime/rpc/key_type_token_mapping.py | 2 +-
 4 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/config/galaxy.yml.realtime b/config/galaxy.yml.realtime
index a55c3439ff4..715a325eedd 100644
--- a/config/galaxy.yml.realtime
+++ b/config/galaxy.yml.realtime
@@ -12,12 +12,12 @@ uwsgi:
 
   module: galaxy.webapps.galaxy.buildapp:uwsgi_app()
 
-  sessions: database/session_map.sqlite
+  realtime_map: database/realtime_map.sqlite
   python-raw: lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
   route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.localhost:8080)$ goto:realtime
   route-run: goto:endendend
   route-label: realtime
-  route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.localhost:8080)$ rpcvar:TARGET_HOST rtt_key_type_token_mapper $2 $1 $3 $4 $0
+  route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.localhost:8080)$ rpcvar:TARGET_HOST rtt_key_type_token_mapper_cached $2 $1 $3 $4 $0 5
   route-if-not: empty:${TARGET_HOST} httpdumb:${TARGET_HOST}
   route-label: endendend
 
diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py
index f35f5a389b7..30368ccec8b 100644
--- a/lib/galaxy/config.py
+++ b/lib/galaxy/config.py
@@ -668,6 +668,8 @@ class Configuration(object):
         self.fluent_log = string_as_bool(kwargs.get('fluent_log', False))
         self.fluent_host = kwargs.get('fluent_host', 'localhost')
         self.fluent_port = int(kwargs.get('fluent_port', 24224))
+        # RealTimeTools propagator mapping file
+        self.realtime_map = self.resolve_path(kwargs.get("realtime_map", "database/realtime_map.sqlite"))
         # directory where the visualization registry searches for plugins
         self.visualization_plugins_directory = kwargs.get(
             'visualization_plugins_directory', 'config/plugins/visualizations')
diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index 1f89b8c4d8c..f0d4d00d4dd 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -149,7 +149,7 @@ class RealTimeManager(object):
         self.security = app.security
         self.sa_session = app.model.context
         self.job_manager = app.job_manager
-        self.propagator = RealtimeSqlite(app.config.proxy_session_map, app.security.encode_id)
+        self.propagator = RealtimeSqlite(app.config.realtime_map, app.security.encode_id)
 
     def create_entry_points(self, job, tool, flush=True):
         for entry in tool.ports:
diff --git a/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py b/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
index a25cc6e0b56..3461c5c5783 100644
--- a/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
+++ b/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
@@ -6,7 +6,7 @@ from time import time
 import uwsgi
 
 
-realtime_db_file = uwsgi.opt["sessions"]
+realtime_db_file = uwsgi.opt["realtime_map"]
 db_conn = sqlite3.connect(realtime_db_file)
 
 

From 2d55589255ae3c8b3ddc362b4f530f9a26819b17 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 22 Mar 2019 15:25:24 -0400
Subject: [PATCH 023/114] move realtime rpc script to /scripts/realtime/

---
 config/galaxy.yml.realtime                                      | 2 +-
 .../realtime/rpc => scripts/realtime}/key_type_token_mapping.py | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename {lib/galaxy/web/realtime/rpc => scripts/realtime}/key_type_token_mapping.py (100%)

diff --git a/config/galaxy.yml.realtime b/config/galaxy.yml.realtime
index 715a325eedd..bb6f5203ff2 100644
--- a/config/galaxy.yml.realtime
+++ b/config/galaxy.yml.realtime
@@ -13,7 +13,7 @@ uwsgi:
   module: galaxy.webapps.galaxy.buildapp:uwsgi_app()
 
   realtime_map: database/realtime_map.sqlite
-  python-raw: lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
+  python-raw: scripts/realtime/key_type_token_mapping.py
   route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.localhost:8080)$ goto:realtime
   route-run: goto:endendend
   route-label: realtime
diff --git a/lib/galaxy/web/realtime/rpc/key_type_token_mapping.py b/scripts/realtime/key_type_token_mapping.py
similarity index 100%
rename from lib/galaxy/web/realtime/rpc/key_type_token_mapping.py
rename to scripts/realtime/key_type_token_mapping.py

From 9ee4b9c4e293099be5728ef0ac155309fe2bd1e7 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 22 Mar 2019 15:27:47 -0400
Subject: [PATCH 024/114] move realtime rpc script to /scripts/realtime/

---
 scripts/realtime/key_type_token_mapping.py | 86 ----------------------
 1 file changed, 86 deletions(-)

diff --git a/scripts/realtime/key_type_token_mapping.py b/scripts/realtime/key_type_token_mapping.py
index 3461c5c5783..e69de29bb2d 100644
--- a/scripts/realtime/key_type_token_mapping.py
+++ b/scripts/realtime/key_type_token_mapping.py
@@ -1,86 +0,0 @@
-import itertools
-import sqlite3
-from threading import RLock
-from time import time
-
-import uwsgi
-
-
-realtime_db_file = uwsgi.opt["realtime_map"]
-db_conn = sqlite3.connect(realtime_db_file)
-
-
-class CacheEntry():
-    def __init__(self, key, value, ttl=20):
-        self.key = key
-        self.value = value
-        self.expires_at = time() + ttl
-        self._expired = False
-
-    def expired(self):
-        if self._expired is False:
-            return (self.expires_at < time())
-        else:
-            return self._expired
-
-
-class CacheList():
-    def __init__(self):
-        self.entries = []
-        self.lock = RLock()
-
-    def add_entry(self, key, value, ttl=20):
-        with self.lock:
-            self.entries.append(CacheEntry(key, value, ttl))
-
-    def read_entries(self):
-        with self.lock:
-            self.entries = list(itertools.dropwhile(lambda x: x.expired(), self.entries))
-            return self.entries
-
-    def get_entry_value(self, key, default=None):
-        entries = self.read_entries()
-        for entry in entries:
-            if entry.key == key:
-                return entry.value
-        return default
-
-
-key_type_token_mapped_cache = CacheList()
-
-
-def key_type_token_mapper_cached(key, key_type, token, route_extra, url, ttl):
-    global key_type_token_mapped_cache
-    cache_key = (key, key_type, token)
-    entry = key_type_token_mapped_cache.get_entry_value(cache_key, None)
-    if entry is None:
-        entry = key_type_token_mapper(key, key_type, token, route_extra, url)
-        if entry is not None:
-            # Should we cache empt/not authorized entries, perhaps for shorter time?
-            key_type_token_mapped_cache.add_entry(cache_key, entry, ttl=float(ttl))
-    return entry
-
-
-def key_type_token_mapper(key, key_type, token, route_extra, url):
-    global db_conn
-    # print 'key %s key_type %s token %s route_extra %s url %s\n' % (key, key_type, token, route_extra, url)
-    if key and key_type and token:
-        # sqlite3.ProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id x and this is thread id y.
-        # So try upto 2 times
-        for i in range(2):
-            # Order by rowid gives us the last row added
-            try:
-                row = db_conn.execute("select host, port from gxrtproxy where key=? and key_type=? and token=? order by rowid desc limit 1", (key, key_type, token)).fetchone()
-                if row:
-                    rval = '%s:%s' % (tuple(row))
-                    return rval.encode()
-                break
-            except sqlite3.ProgrammingError:
-                db_conn = sqlite3.connect(realtime_db_file)
-                continue
-            break
-    return None
-
-
-uwsgi.register_rpc('rtt_key_type_token_mapper', key_type_token_mapper)
-uwsgi.register_rpc('rtt_key_type_token_mapper_cached', key_type_token_mapper_cached)

From 0a6e4bbda5fe92487ca3b65df13d86ab4f94c7ee Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 22 Mar 2019 15:29:58 -0400
Subject: [PATCH 025/114] Revert "move realtime rpc script to
 /scripts/realtime/"

This reverts commit 9ee4b9c4e293099be5728ef0ac155309fe2bd1e7.
---
 scripts/realtime/key_type_token_mapping.py | 86 ++++++++++++++++++++++
 1 file changed, 86 insertions(+)

diff --git a/scripts/realtime/key_type_token_mapping.py b/scripts/realtime/key_type_token_mapping.py
index e69de29bb2d..3461c5c5783 100644
--- a/scripts/realtime/key_type_token_mapping.py
+++ b/scripts/realtime/key_type_token_mapping.py
@@ -0,0 +1,86 @@
+import itertools
+import sqlite3
+from threading import RLock
+from time import time
+
+import uwsgi
+
+
+realtime_db_file = uwsgi.opt["realtime_map"]
+db_conn = sqlite3.connect(realtime_db_file)
+
+
+class CacheEntry():
+    def __init__(self, key, value, ttl=20):
+        self.key = key
+        self.value = value
+        self.expires_at = time() + ttl
+        self._expired = False
+
+    def expired(self):
+        if self._expired is False:
+            return (self.expires_at < time())
+        else:
+            return self._expired
+
+
+class CacheList():
+    def __init__(self):
+        self.entries = []
+        self.lock = RLock()
+
+    def add_entry(self, key, value, ttl=20):
+        with self.lock:
+            self.entries.append(CacheEntry(key, value, ttl))
+
+    def read_entries(self):
+        with self.lock:
+            self.entries = list(itertools.dropwhile(lambda x: x.expired(), self.entries))
+            return self.entries
+
+    def get_entry_value(self, key, default=None):
+        entries = self.read_entries()
+        for entry in entries:
+            if entry.key == key:
+                return entry.value
+        return default
+
+
+key_type_token_mapped_cache = CacheList()
+
+
+def key_type_token_mapper_cached(key, key_type, token, route_extra, url, ttl):
+    global key_type_token_mapped_cache
+    cache_key = (key, key_type, token)
+    entry = key_type_token_mapped_cache.get_entry_value(cache_key, None)
+    if entry is None:
+        entry = key_type_token_mapper(key, key_type, token, route_extra, url)
+        if entry is not None:
+            # Should we cache empt/not authorized entries, perhaps for shorter time?
+            key_type_token_mapped_cache.add_entry(cache_key, entry, ttl=float(ttl))
+    return entry
+
+
+def key_type_token_mapper(key, key_type, token, route_extra, url):
+    global db_conn
+    # print 'key %s key_type %s token %s route_extra %s url %s\n' % (key, key_type, token, route_extra, url)
+    if key and key_type and token:
+        # sqlite3.ProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id x and this is thread id y.
+        # So try upto 2 times
+        for i in range(2):
+            # Order by rowid gives us the last row added
+            try:
+                row = db_conn.execute("select host, port from gxrtproxy where key=? and key_type=? and token=? order by rowid desc limit 1", (key, key_type, token)).fetchone()
+                if row:
+                    rval = '%s:%s' % (tuple(row))
+                    return rval.encode()
+                break
+            except sqlite3.ProgrammingError:
+                db_conn = sqlite3.connect(realtime_db_file)
+                continue
+            break
+    return None
+
+
+uwsgi.register_rpc('rtt_key_type_token_mapper', key_type_token_mapper)
+uwsgi.register_rpc('rtt_key_type_token_mapper_cached', key_type_token_mapper_cached)

From c484330c23fc23d941a3a8fb2ce9963fec312d90 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Wed, 27 Mar 2019 11:48:51 -0400
Subject: [PATCH 026/114] rename rtt db migration

---
 .../{0150_add_realtimetools.py => 0151_add_realtimetools.py}      | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename lib/galaxy/model/migrate/versions/{0150_add_realtimetools.py => 0151_add_realtimetools.py} (100%)

diff --git a/lib/galaxy/model/migrate/versions/0150_add_realtimetools.py b/lib/galaxy/model/migrate/versions/0151_add_realtimetools.py
similarity index 100%
rename from lib/galaxy/model/migrate/versions/0150_add_realtimetools.py
rename to lib/galaxy/model/migrate/versions/0151_add_realtimetools.py

From 3e944027ab8146813e231ef9f2db2d589d0f651f Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 2 Apr 2019 16:32:59 -0400
Subject: [PATCH 027/114] rename realtimetool migration to higher version

---
 .../{0151_add_realtimetools.py => 0152_add_realtimetools.py}      | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename lib/galaxy/model/migrate/versions/{0151_add_realtimetools.py => 0152_add_realtimetools.py} (100%)

diff --git a/lib/galaxy/model/migrate/versions/0151_add_realtimetools.py b/lib/galaxy/model/migrate/versions/0152_add_realtimetools.py
similarity index 100%
rename from lib/galaxy/model/migrate/versions/0151_add_realtimetools.py
rename to lib/galaxy/model/migrate/versions/0152_add_realtimetools.py

From 054e36edf1f60566a66de14b71064a91c324ef2f Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 8 Apr 2019 10:44:52 -0400
Subject: [PATCH 028/114] change version number of realtimetools migration

---
 .../{0152_add_realtimetools.py => 0153_add_realtimetools.py}      | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename lib/galaxy/model/migrate/versions/{0152_add_realtimetools.py => 0153_add_realtimetools.py} (100%)

diff --git a/lib/galaxy/model/migrate/versions/0152_add_realtimetools.py b/lib/galaxy/model/migrate/versions/0153_add_realtimetools.py
similarity index 100%
rename from lib/galaxy/model/migrate/versions/0152_add_realtimetools.py
rename to lib/galaxy/model/migrate/versions/0153_add_realtimetools.py

From 557135b7f4f0caa0b7343d534b0775f282fcc939 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 8 Apr 2019 16:54:14 -0400
Subject: [PATCH 029/114] update migrate version of realtimetools

---
 .../{0153_add_realtimetools.py => 0154_add_realtimetools.py}      | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename lib/galaxy/model/migrate/versions/{0153_add_realtimetools.py => 0154_add_realtimetools.py} (100%)

diff --git a/lib/galaxy/model/migrate/versions/0153_add_realtimetools.py b/lib/galaxy/model/migrate/versions/0154_add_realtimetools.py
similarity index 100%
rename from lib/galaxy/model/migrate/versions/0153_add_realtimetools.py
rename to lib/galaxy/model/migrate/versions/0154_add_realtimetools.py

From ee680d7c4599f4da1518194377ac51d0b9e80bd5 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 16 Apr 2019 15:40:50 -0400
Subject: [PATCH 030/114] Remove inspection check, only do ports

---
 lib/galaxy/jobs/runners/local.py | 25 ++-----------------------
 1 file changed, 2 insertions(+), 23 deletions(-)

diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py
index 9b87296ea44..dab4d213b8a 100644
--- a/lib/galaxy/jobs/runners/local.py
+++ b/lib/galaxy/jobs/runners/local.py
@@ -263,31 +263,10 @@ class LocalJobRunner(BaseJobRunner):
             cont = job.container
             if cont:
                 if cont.container_type == 'docker':
-                    # Do we need to do inspect, or is only ports ok?
-                    max_command_attempts = 10 + 1
-                    inspect = None
-                    exit_code = None
-                    for i in range(1, max_command_attempts):
-                        with tempfile.TemporaryFile() as stdout_file:
-                            exit_code = subprocess.call(cont.container_info['commands']['inspect'],
-                                                    shell=True,
-                                                    stdout=stdout_file,
-                                                    env=self._environ,
-                                                    preexec_fn=os.setpgrp)
-                            if exit_code == 0:
-                                stdout_file.seek(0)
-                                inspect = stdout_file.read()
-                                break
-                        if i != max_command_attempts:
-                            t = 2 * i
-                            log.debug("Container not found during inspect check, sleeping for %s seconds.", t)
-                            sleep(t)
-                    if inspect:
-                        cont.container_info['inspect'] = json.loads(inspect)
-                    else:
-                        log.error('Unable to run inspect command (%s): %s', cont.container_info['commands']['inspect'], exit_code)
+                    # Handle getting RealTimeTool ports, if applicable
                     eps = job.realtimetool_entry_points
                     if eps:
+                        max_command_attempts = 10 + 1
                         ports_raw = None
                         for i in range(1, max_command_attempts):
                             with tempfile.TemporaryFile() as stdout_file:

From 7a5a98701b55003d2bf0e9d35cc076707eaaf674 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 16 Apr 2019 15:49:51 -0400
Subject: [PATCH 031/114] Remove unneeded RealTimeTool datatype.

---
 client/galaxy/scripts/mvc/tool/tool-form.js |  3 +-
 config/datatypes_conf.xml.sample            |  1 -
 lib/galaxy/datatypes/realtime.py            | 77 ---------------------
 3 files changed, 1 insertion(+), 80 deletions(-)
 delete mode 100644 lib/galaxy/datatypes/realtime.py

diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js
index 8a2170db72e..bbde1898662 100644
--- a/client/galaxy/scripts/mvc/tool/tool-form.js
+++ b/client/galaxy/scripts/mvc/tool/tool-form.js
@@ -353,8 +353,7 @@ var View = Backbone.View.extend({
                 vr += `<div class="infomessagelarge">
                             <p>
                                 <a href="${response.view_result[i]}">
-                                There is a RealTimeTool result view available, click here to display. You can also click on the display eye icon (<span class="fa fa-eye"></span>)
-                                associated with the RealTimeTool Dataset to view.</a>
+                                There is a RealTimeTool result view available, click here to display.</a>
                             </p>
                             <p>
                                 You may also access all active RealTimeTools from the User menu.
diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample
index ccf5fd6abc2..e3a260b6a05 100644
--- a/config/datatypes_conf.xml.sample
+++ b/config/datatypes_conf.xml.sample
@@ -172,7 +172,6 @@
       <display file="rviewer/bed.xml" inherit="true"/>
       <display file="igv/interval_as_bed.xml" inherit="true"/>
     </datatype>
-    <datatype extension="realtimetool" type="galaxy.datatypes.realtime:Realtime" description="Realtime Tool Output" mimetype="text/html" display_in_upload="false"/>
 
     <!-- ISA data types -->
     <datatype extension="isa-tab" type="galaxy.datatypes.isa:IsaTab" mimetype="application/isa-tools" display_in_upload="true" description="ISA-Tab data type." description_url="https://isa-tools.org"/>
diff --git a/lib/galaxy/datatypes/realtime.py b/lib/galaxy/datatypes/realtime.py
deleted file mode 100644
index 1cc9847affd..00000000000
--- a/lib/galaxy/datatypes/realtime.py
+++ /dev/null
@@ -1,77 +0,0 @@
-"""Binary classes"""
-from __future__ import print_function
-
-import logging
-import os
-import sys
-
-from galaxy.web import url_for
-from .text import Html
-
-
-log = logging.getLogger(__name__)
-
-
-class Realtime(Html):
-    """Class describing a RealtimeTool Default output file"""
-    file_ext = "realtimetool"
-    composite_type = 'auto_primary_file'
-
-    def set_peek(self, dataset, is_multi_byte=False):
-        dataset.blurb = "RealtimeTool"
-        dataset.peek = 'Finished RealTimeTool'
-
-    def display_peek(self, dataset):
-        try:
-            if not dataset.peek:
-                self.set_peek(dataset)
-            return dataset.peek
-        except Exception:
-            return "Realtime Tool file"
-
-    def generate_primary_file(self, dataset=None):
-        """
-        This is called only at creation to write the html file
-        """
-        defined_files = self.get_composite_files(dataset=dataset).items()
-        rval = ["<html><head><title>Files for Composite Dataset (%s)</title></head>" % (self.file_ext)]
-        if defined_files:
-            rval.append("<p/>This composite dataset is composed of the following defined files:<p/><ul>")
-            for composite_name, composite_file in defined_files:
-                opt_text = ''
-                if composite_file.optional:
-                    opt_text = ' (optional)'
-                missing_text = ''
-                if not os.path.exists(os.path.join(dataset.extra_files_path, composite_name)):
-                    missing_text = ' (missing)'
-                rval.append('<li><a href="%s">%s</a>%s%s</li>' % (composite_name, composite_name, opt_text, missing_text))
-            rval.append("</ul>")
-        defined_files = map(lambda x: x[0], defined_files)
-        extra_files = []
-        for (dirpath, dirnames, filenames) in os.walk(dataset.extra_files_path, followlinks=True):
-            for filename in filenames:
-                rel_path = os.path.relpath(os.path.join(dirpath, filename), dataset.extra_files_path)
-                if rel_path not in defined_files:
-                    extra_files.append(rel_path)
-        if extra_files:
-            rval.append("<p/>This composite dataset contains these undefined files:<p/><ul>")
-            for rel_path in extra_files:
-                rval.append('<li><a href="%s">%s</a></li>' % (rel_path, rel_path))
-            rval.append('</ul>')
-        if not (defined_files or extra_files):
-            rval.append("<p/>This composite dataset does not contain any files!<p/><ul>")
-        rval.append('</html>')
-        return "\n".join(rval)
-
-    def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, offset=None, ck_size=None, **kwd):
-        job = dataset.creating_job
-        if job and not job.finished:
-            eps = job.realtimetool_entry_points
-            if eps:
-                return trans.response.send_redirect(url_for(controller='realtime', action='index', entry_point_id=[trans.security.encode_id(ep.id) for ep in eps]))
-        return super(Realtime, self).display_data(trans, dataset, preview=preview, filename=filename, to_ext=to_ext, offset=offset, ck_size=ck_size, **kwd)
-
-
-if __name__ == '__main__':
-    import doctest
-    doctest.testmod(sys.modules[__name__])

From 2fa9ff185ea1f0ccc857ce83690d4d1b322330d0 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 16 Apr 2019 16:52:12 -0400
Subject: [PATCH 032/114] No more inspect, no more json needed.

---
 lib/galaxy/jobs/runners/local.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py
index dab4d213b8a..85e5a0333a3 100644
--- a/lib/galaxy/jobs/runners/local.py
+++ b/lib/galaxy/jobs/runners/local.py
@@ -3,7 +3,6 @@ Job runner plugin for executing jobs on the local system via the command line.
 """
 import datetime
 import errno
-import json
 import logging
 import os
 import subprocess

From e5947b950a4e93436876f632639384fbe75aff7e Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Thu, 9 May 2019 15:23:32 -0400
Subject: [PATCH 033/114] Update rtt migrate version number

---
 .../{0154_add_realtimetools.py => 0155_add_realtimetools.py}      | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename lib/galaxy/model/migrate/versions/{0154_add_realtimetools.py => 0155_add_realtimetools.py} (100%)

diff --git a/lib/galaxy/model/migrate/versions/0154_add_realtimetools.py b/lib/galaxy/model/migrate/versions/0155_add_realtimetools.py
similarity index 100%
rename from lib/galaxy/model/migrate/versions/0154_add_realtimetools.py
rename to lib/galaxy/model/migrate/versions/0155_add_realtimetools.py

From 97e8a4fdbb437261e0a59c432ef5c1c8ca15eba4 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Thu, 9 May 2019 16:21:24 -0400
Subject: [PATCH 034/114] js-lint fix

---
 client/galaxy/scripts/mvc/tool/tool-form.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js
index dee6f75e8c9..43d2bf88128 100644
--- a/client/galaxy/scripts/mvc/tool/tool-form.js
+++ b/client/galaxy/scripts/mvc/tool/tool-form.js
@@ -347,7 +347,7 @@ var View = Backbone.View.extend({
     _getViewResult: function(response) {
         var vr = "";
         if (response.view_result) {
-            for (let i in response.view_result) {
+            for (const i in response.view_result) {
                 vr += `<div class="infomessagelarge">
                             <p>
                                 <a href="${response.view_result[i]}">

From 1fdfdd39decd070fe68fadcabc3824f232c9cc69 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 10 May 2019 17:00:21 -0400
Subject: [PATCH 035/114] remove extra _

---
 lib/galaxy/jobs/runners/local.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py
index 85e5a0333a3..45674d2ce51 100644
--- a/lib/galaxy/jobs/runners/local.py
+++ b/lib/galaxy/jobs/runners/local.py
@@ -181,7 +181,7 @@ class LocalJobRunner(BaseJobRunner):
                 return
         else:
             log.warning("stop_job(): %s: PID %d refuses to die after signaling TERM/KILL" % (job.id, pid))
-        self.__kill_container(job_wrapper)
+        self._kill_container(job_wrapper)
 
     def recover(self, job, job_wrapper):
         # local jobs can't be recovered

From 3d829c409fa3e6da4fa5772b03c9e151a1f66ab7 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 13 May 2019 10:13:43 -0400
Subject: [PATCH 036/114] Update RealTimeTool entry point definitions.

---
 lib/galaxy/tools/__init__.py         |  2 +-
 lib/galaxy/tools/parser/interface.py |  4 ++--
 lib/galaxy/tools/parser/xml.py       | 24 +++++++++++++-----------
 lib/galaxy/tools/parser/yaml.py      |  4 ++--
 4 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index c2832b1d674..2ccf21e1f82 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -817,7 +817,7 @@ class Tool(Dictifiable):
         self.__parse_trackster_conf(tool_source)
         # Record macro paths so we can reload a tool if any of its macro has changes
         self._macro_paths = tool_source.macro_paths()
-        self.ports = tool_source.parse_ports()
+        self.ports = tool_source.parse_realtime()
 
     def __parse_legacy_features(self, tool_source):
         self.code_namespace = dict()
diff --git a/lib/galaxy/tools/parser/interface.py b/lib/galaxy/tools/parser/interface.py
index 1fcca1e0c34..77ce784250e 100644
--- a/lib/galaxy/tools/parser/interface.py
+++ b/lib/galaxy/tools/parser/interface.py
@@ -123,8 +123,8 @@ class ToolSource(object):
         """
 
     @abstractmethod
-    def parse_ports(self):
-        """ Return port templates to expose.
+    def parse_realtime(self):
+        """ Return RealTimeTool entry point templates to expose.
         """
 
     def parse_redirect_url_params_elem(self):
diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py
index 80a4f43fca6..c1a1d20c9ba 100644
--- a/lib/galaxy/tools/parser/xml.py
+++ b/lib/galaxy/tools/parser/xml.py
@@ -195,21 +195,23 @@ class XmlToolSource(ToolSource):
             return ParallelismInfo(parallelism)
         return parallelism_info
 
-    def parse_ports(self):
-        ports_el = self.root.find("ports")
-        ports = []
-        if ports_el is None:
-            return ports
-        for port_el in ports_el.findall("port"):
-            port = port_el.text.strip()
-            url = port_el.find("url")
+    def parse_realtime(self):
+        realtimetool_el = self.root.find("realtime")
+        rtt = []
+        if realtimetool_el is None:
+            return rtt
+        for ep_el in realtimetool_el.findall("entry_point"):
+            port = ep_el.find("port")
+            assert port is not None, ValueError('A port is required for RealTimeTools')
+            port = port.text.strip()
+            url = ep_el.find("url")
             if url is not None:
                 url = url.text.strip()
-            name = port_el.get('name', None)
+            name = ep_el.get('name', None)
             if name:
                 name = name.strip()
-            ports.append(dict(port=port, url=url, name=name))
-        return ports
+            rtt.append(dict(port=port, url=url, name=name))
+        return rtt
 
     def parse_hidden(self):
         hidden = xml_text(self.root, "hidden")
diff --git a/lib/galaxy/tools/parser/yaml.py b/lib/galaxy/tools/parser/yaml.py
index de44cc392d8..78e408fe8e8 100644
--- a/lib/galaxy/tools/parser/yaml.py
+++ b/lib/galaxy/tools/parser/yaml.py
@@ -164,8 +164,8 @@ class YamlToolSource(ToolSource):
     def parse_profile(self):
         return self.root_dict.get("profile", "16.04")
 
-    def parse_ports(self):
-        return self.root_dict.get("ports", [])
+    def parse_realtime(self):
+        return self.root_dict.get("realtime", [])
 
     def parse_python_template_version(self):
         python_template_version = self.root_dict.get("python_template_version", None)

From 0c943508ae32096b17aa5093dca6374f7922cc79 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 13 May 2019 10:23:38 -0400
Subject: [PATCH 037/114] Update xsd

---
 lib/galaxy/tools/xsd/galaxy.xsd | 43 +++++++++++++++++++++------------
 1 file changed, 28 insertions(+), 15 deletions(-)

diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd
index 5f641a35b1a..91d55e1efe5 100644
--- a/lib/galaxy/tools/xsd/galaxy.xsd
+++ b/lib/galaxy/tools/xsd/galaxy.xsd
@@ -48,7 +48,7 @@ A ``data_source`` tool contains a few more relevant attributes.
         <xs:element name="edam_topics" type="EdamTopics" minOccurs="0"/>
         <xs:element name="edam_operations" type="EdamOperations" minOccurs="0"/>
         <xs:element name="requirements" type="Requirements" minOccurs="0"/>
-        <xs:element name="ports" type="Ports" minOccurs="0" maxOccurs="1" />
+        <xs:element name="realtime" type="RealTime" minOccurs="0" maxOccurs="1" />
         <xs:element name="description" type="xs:string" minOccurs="0">
           <xs:annotation gxdocs:best_practices="tool-descriptions">
             <xs:documentation xml:lang="en"><![CDATA[The value is displayed in
@@ -225,39 +225,43 @@ For more information, see https://planemo.readthedocs.io/en/latest/writing_advan
     </xs:sequence>
   </xs:complexType>
 
-  <xs:complexType name="Ports">
+  <xs:complexType name="RealTime">
     <xs:annotation>
       <xs:documentation xml:lang="en"><![CDATA[
 
-This is a container tag set for the ``port`` and ``url`` tags
-described in greater detail below. ``ports``s describe RealTimeTool entry points
+This is a container tag set for the ``entry_point`` tag that contains ``port`` and ``url`` tags
+described in greater detail below. ``entry_point``s describe RealTimeTool entry points
 to a tool.
 
 ]]></xs:documentation>
     </xs:annotation>
     <xs:sequence>
-      <xs:element name="port" type="Port" minOccurs="0" maxOccurs="unbounded"/>
+      <xs:element name="entry_point" type="EntryPoint" minOccurs="0" maxOccurs="unbounded"/>
     </xs:sequence>
   </xs:complexType>
 
-  <xs:complexType name="Port" mixed="true">
+  <xs:complexType name="EntryPoint" mixed="true">
     <xs:annotation>
       <xs:documentation xml:lang="en"><![CDATA[
 
-This tag set is contained within the ``<ports>`` tag set. Access ports and
-entry point urls are included in this tag set. These are used by RealTimeTools
-to provide access to graphical tools in real time.
+This tag set is contained within the ``<entry_point>`` tag set. Access to entry point
+ports and urls are included in this tag set. These are used by RealTimeTools
+to provide access to graphical tools in real-time.
 
 ```xml
-<ports>
-    <port name="Example name">80<url>landing/index.html</url></port>
-</ports>
+<realtime>
+  <entry_point name="Example name">
+    <port>80</port>
+    <url>landing/${template_enabled}/index.html</url>
+  </entry_point>
+</realtime>
 ```
 
 ]]></xs:documentation>
     </xs:annotation>
     <xs:sequence>
-      <xs:element name="url" type="PortURL" minOccurs="0" maxOccurs="1"/>
+      <xs:element name="port" type="EntryPointPort" minOccurs="1" maxOccurs="1"/>
+      <xs:element name="url" type="EntryPointURL" minOccurs="0" maxOccurs="1"/>
     </xs:sequence>
         <xs:attribute name="name" type="xs:string" use="required">
           <xs:annotation>
@@ -265,11 +269,20 @@ to provide access to graphical tools in real time.
           </xs:annotation>
         </xs:attribute>
   </xs:complexType>
-  <xs:complexType name="PortURL">
+  <xs:complexType name="EntryPointPort" mixed="true">
     <xs:annotation>
       <xs:documentation xml:lang="en"><![CDATA[
 
-This tag set is contained within the ``<port>`` tag set. It contains the entry URL.
+This tag set is contained within the ``<entry_point>`` tag set. It contains the entry port.
+
+]]></xs:documentation>
+    </xs:annotation>
+  </xs:complexType>
+  <xs:complexType name="EntryPointURL" mixed="true">
+    <xs:annotation>
+      <xs:documentation xml:lang="en"><![CDATA[
+
+This tag set is contained within the ``<entry_point>`` tag set. It contains the entry URL.
 
 ]]></xs:documentation>
     </xs:annotation>

From ac7d6fd9749bcf9b270f447060bc0e41cab0e182 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 13 May 2019 14:16:10 -0400
Subject: [PATCH 038/114] Enable templating of RealTime tools name, port, url.

---
 lib/galaxy/jobs/__init__.py          |  1 +
 lib/galaxy/jobs/runners/__init__.py  |  3 +--
 lib/galaxy/managers/realtime.py      |  9 +++++----
 lib/galaxy/tools/__init__.py         |  5 +----
 lib/galaxy/tools/actions/realtime.py | 15 ---------------
 lib/galaxy/tools/evaluation.py       | 24 ++++++++++++++++++++++++
 6 files changed, 32 insertions(+), 25 deletions(-)
 delete mode 100644 lib/galaxy/tools/actions/realtime.py

diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py
index a868c1bcecb..a3453123d9b 100644
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -1046,6 +1046,7 @@ class JobWrapper(HasResourceParameters):
         # if the server was stopped and restarted before the job finished
         job.command_line = unicodify(self.command_line)
         job.dependencies = self.tool.dependencies
+        self.realtimetools = tool_evaluator.realtimetools
         self.sa_session.add(job)
         self.sa_session.flush()
         # Return list of all extra files
diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py
index e1cddcbae1e..b9d6259bdb7 100644
--- a/lib/galaxy/jobs/runners/__init__.py
+++ b/lib/galaxy/jobs/runners/__init__.py
@@ -417,8 +417,7 @@ class BaseJobRunner(object):
             compute_tmp_directory = job_wrapper.tmp_directory()
 
         tool = job_wrapper.tool
-        # TODO: template RealTimeTool ports?
-        guest_ports = [port.get('port') for port in tool.ports]
+        guest_ports = [ep.get('port') for ep in job_wrapper.realtimetools]
         tool_info = ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through, guest_ports=guest_ports)
         job_info = JobInfo(
             compute_working_directory,
diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index f0d4d00d4dd..38ec7dfb62b 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -151,8 +151,9 @@ class RealTimeManager(object):
         self.job_manager = app.job_manager
         self.propagator = RealtimeSqlite(app.config.realtime_map, app.security.encode_id)
 
-    def create_entry_points(self, job, tool, flush=True):
-        for entry in tool.ports:
+    def create_entry_points(self, job, tool, entry_points=None, flush=True):
+        entry_points = entry_points or tool.ports
+        for entry in entry_points:
             ep = self.model.RealTimeToolEntryPoint(job=job, tool_port=entry['port'], entry_url=entry['url'], name=entry['name'])
             self.sa_session.add(ep)
         if flush:
@@ -192,10 +193,10 @@ class RealTimeManager(object):
         """
         self.propagator.save_entry_point(entry_point)
 
-    def create_realtime(self, job, tool):
+    def create_realtime(self, job, tool, entry_points):
         # create from initial job
         if job and tool:
-            self.create_entry_points(job, tool)
+            self.create_entry_points(job, tool, entry_points)
         else:
             log.warning('Called RealTimeManager.create_realtime, but job (%s) or tool (%s) is None', job, tool)
 
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 2ccf21e1f82..dd012578d2a 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -2444,7 +2444,6 @@ class ImportHistoryTool(Tool):
 
 class RealTimeTool(Tool):
     tool_type = 'realtime'
-    default_tool_action = RealTimeToolAction
 
     def __remove_realtime_by_job(self, job):
         if job:
@@ -2466,9 +2465,7 @@ class RealTimeTool(Tool):
 
     def get_view_result(self, job=None):
         if job:
-            eps = job.realtimetool_entry_points
-            if eps:
-                return self.app.url_for(controller='realtime', action='index', entry_point_id=[self.app.security.encode_id(ep.id) for ep in eps])
+            return self.app.url_for(controller='realtime', action='index', job_id=self.app.security.encode_id(job.id))
         return None
 
 
diff --git a/lib/galaxy/tools/actions/realtime.py b/lib/galaxy/tools/actions/realtime.py
deleted file mode 100644
index b40c717b993..00000000000
--- a/lib/galaxy/tools/actions/realtime.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import logging
-
-from . import DefaultToolAction
-
-log = logging.getLogger(__name__)
-
-
-class RealTimeToolAction(DefaultToolAction):
-    """Tool action used for RealTime Tools"""
-
-    def execute(self, tool, trans, **kwds):
-        rval = super(RealTimeToolAction, self).execute(tool, trans, **kwds)
-        job, output_dict = rval
-        trans.app.realtime_manager.create_realtime(job, tool)
-        return rval
diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py
index 162f0c906d1..6aff14b7fb9 100644
--- a/lib/galaxy/tools/evaluation.py
+++ b/lib/galaxy/tools/evaluation.py
@@ -149,6 +149,8 @@ class ToolEvaluator(object):
         self.__sanitize_param_dict(param_dict)
         # Parameters added after this line are not sanitized
         self.__populate_non_job_params(param_dict)
+        # Populate and store templated RealTimeTools values
+        self.__populate_realtimetools(param_dict)
 
         # Return the dictionary of parameters
         return param_dict
@@ -404,6 +406,28 @@ class ToolEvaluator(object):
             # the paths rewritten.
             self.__walk_inputs(self.tool.inputs, param_dict, rewrite_unstructured_paths)
 
+    def __populate_realtimetools(self, param_dict):
+        """
+        Populate RealTimeTools templated values.
+        """
+        rtt = []
+        for ep in self.tool.ports:
+            ep_dict = {}
+            for key in 'port', 'name', 'url':
+                val = ep.get(key, None)
+                if val is not None:
+                    val = fill_template(val, context=param_dict, python_template_version=self.tool.python_template_version)
+                    clean_val = []
+                    for line in val.split('\n'):
+                        clean_val.append(line.strip())
+                    val = '\n'.join(clean_val)
+                    val = val.replace("\n", " ").replace("\r", " ").strip()
+                ep_dict[key] = val
+            rtt.append(ep_dict)
+        self.realtimetools = rtt
+        self.app.realtime_manager.create_realtime(self.job, self.tool, rtt)
+        return rtt
+
     def __sanitize_param_dict(self, param_dict):
         """
         Sanitize all values that will be substituted on the command line, with the exception of ToolParameterValueWrappers,

From 7376cc9fd78d9edad2a9e9500fb8c927d881fcba Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 13 May 2019 14:36:31 -0400
Subject: [PATCH 039/114] Add docs and example job config.

---
 config/job_conf.xml.realtime                  | 27 ++++++
 .../admin/special_topics/realtimetools.rst    | 93 +++++++++++++++++++
 2 files changed, 120 insertions(+)
 create mode 100644 config/job_conf.xml.realtime
 create mode 100644 doc/source/admin/special_topics/realtimetools.rst

diff --git a/config/job_conf.xml.realtime b/config/job_conf.xml.realtime
new file mode 100644
index 00000000000..8cf4ee48117
--- /dev/null
+++ b/config/job_conf.xml.realtime
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!-- A sample job config for RealTimeTools using local runner. -->
+<job_conf>
+    <plugins>
+        <plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="4"/>
+    </plugins>
+    <destinations default="docker_dispatch">
+        <destination id="local" runner="local"/>
+
+        <destination id="docker_local" runner="local">
+          <param id="docker_enabled">true</param>
+          <param id="docker_volumes">$galaxy_root:ro,$tool_directory:ro,$job_directory:rw,$working_directory:rw,$default_file_path:ro</param>
+          <param id="docker_sudo">false</param>
+          <param id="docker_net">bridge</param>
+          <param id="docker_auto_rm">true</param>
+          <param id="require_container">true</param>
+        </destination>
+
+        <destination id="docker_dispatch" runner="dynamic">
+            <param id="type">docker_dispatch</param>
+            <param id="docker_destination_id">docker_local</param>
+            <param id="default_destination_id">local</param>
+        </destination>
+
+    </destinations>
+
+</job_conf> 
diff --git a/doc/source/admin/special_topics/realtimetools.rst b/doc/source/admin/special_topics/realtimetools.rst
new file mode 100644
index 00000000000..8c532764dba
--- /dev/null
+++ b/doc/source/admin/special_topics/realtimetools.rst
@@ -0,0 +1,93 @@
+Galaxy RealTimeTools
+=====================================
+
+A Galaxy RealTimeTool allows launching a container-backed Galaxy Tool 
+and enabling a Galaxy User to gain access to content inside in real-time.
+
+
+How Galaxy RealTimeTools Work
+-----------------------------
+
+A RealTimeTool is defined in the same familiar way as standard Galaxy Tools,
+but are specified with **tool_type="realtime"**, and providing additional entry point
+information:
+
+.. code-block:: xml
+
+        <realtime>
+            <entry_point name="Display name">
+                <port>80</port>
+                <url><![CDATA[optional/path/can/be/${templated}]]></url>
+            </entry_point>
+        </realtime>
+
+Note that name, port, and url are each able to be templated from the RealTimeTool's parameter dictionary.
+
+
+
+Some important benefits of using Galaxy RealTimeTools
+-----------------------------------------------------
+
+- You can have and access any number of RealTimeTools at a time (admin configurable)
+- If you accidentally close the RealTimeTool browser window, you can regain access by selecting from a list of active RealTimeTools
+- A single RealTimeTool can grant access to multiple running applications, servers,  and interfaces
+- RealTimeTools can be added to Galaxy Workflows
+- Native, out-of-the box support for RealTimeTools in Galaxy via uWSGI proxying
+- RealTimeTools are bonafide Galaxy Tools; just specify tool_type as "realtime" and list the ports you want to expose 
+- RealTimeTools can be added to and installed from the ToolShed.
+- R Shiny apps, Javascript-based VNC access to desktop environments, genome-browsers-in-a-box, interactive notebook environments, etc, are all possible with RealTimeTools
+
+
+
+Server configuration of Galaxy RealTimeTools
+--------------------------------------------
+
+The **galaxy.yml** file will need to be populated as seen in config/galaxy.yml.realtime.
+
+In the **uwsgi:** section:
+
+.. code-block:: yaml
+  http-raw-body: true
+  # master: true
+
+  realtime_map: database/realtime_map.sqlite
+  python-raw: scripts/realtime/key_type_token_mapping.py
+  route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.localhost:8080)$ goto:realtime
+  route-run: goto:endendend
+  route-label: realtime
+  route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.localhost:8080)$ rpcvar:TARGET_HOST rtt_key_type_token_mapper_cached $2 $1 $3 $4 $0 5
+  route-if-not: empty:${TARGET_HOST} httpdumb:${TARGET_HOST}
+  route-label: endendend
+
+In the **galaxy:** section:
+
+.. code-block:: yaml
+
+  realtime_prefix: realtime
+
+
+An example **job_conf.xml** file as seen in config/galaxy.yml.realtime:
+
+.. code-block:: xml
+
+        <job_conf>
+            <plugins>
+                <plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="4"/>
+            </plugins>
+            <destinations default="docker_dispatch">
+                <destination id="local" runner="local"/>
+                <destination id="docker_local" runner="local">
+                  <param id="docker_enabled">true</param>
+                  <param id="docker_volumes">$galaxy_root:ro,$tool_directory:ro,$job_directory:rw,$working_directory:rw,$default_file_path:ro</param>
+                  <param id="docker_sudo">false</param>
+                  <param id="docker_net">bridge</param>
+                  <param id="docker_auto_rm">true</param>
+                  <param id="require_container">true</param>
+                </destination>
+                <destination id="docker_dispatch" runner="dynamic">
+                    <param id="type">docker_dispatch</param>
+                    <param id="docker_destination_id">docker_local</param>
+                    <param id="default_destination_id">local</param>
+                </destination>
+            </destinations>
+        </job_conf> 

From 13a5524308e852c94d8af0fff114e401a631cde9 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 13 May 2019 14:38:15 -0400
Subject: [PATCH 040/114] Remove unused import

---
 lib/galaxy/tools/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index dd012578d2a..1ba5e8c12c4 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -34,7 +34,6 @@ from galaxy.tools.actions import DefaultToolAction
 from galaxy.tools.actions.data_manager import DataManagerToolAction
 from galaxy.tools.actions.data_source import DataSourceToolAction
 from galaxy.tools.actions.model_operations import ModelOperationToolAction
-from galaxy.tools.actions.realtime import RealTimeToolAction
 from galaxy.tools.deps import (
     CachedDependencyManager,
 )

From 718f5f048cd2964b7b482e79286d7a148b66a33a Mon Sep 17 00:00:00 2001
From: Bjoern Gruening <bjoern.gruening@gmail.com>
Date: Sun, 5 May 2019 17:54:19 +0200
Subject: [PATCH 041/114] change cellxgene Container to work with the IE tools

---
 .../interactive_environments/cellxgene/docker/Dockerfile      | 4 ++--
 .../cellxgene/docker/run_cellxgene.sh                         | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/config/plugins/interactive_environments/cellxgene/docker/Dockerfile b/config/plugins/interactive_environments/cellxgene/docker/Dockerfile
index b172e4a0ec5..1d7295f01bd 100644
--- a/config/plugins/interactive_environments/cellxgene/docker/Dockerfile
+++ b/config/plugins/interactive_environments/cellxgene/docker/Dockerfile
@@ -1,9 +1,9 @@
 FROM python:3.6.8-slim
  
-RUN pip3 install cellxgene && rm -rf ~/.cache
+RUN pip3 install cellxgene scanpy==1.4 && rm -rf ~/.cache
 
 ADD ./run_cellxgene.sh /
 
-ENTRYPOINT ["/run_cellxgene.sh"]
+##ENTRYPOINT ["/run_cellxgene.sh"]
 
 EXPOSE 80
diff --git a/config/plugins/interactive_environments/cellxgene/docker/run_cellxgene.sh b/config/plugins/interactive_environments/cellxgene/docker/run_cellxgene.sh
index 0759da7af8e..cb9413d7bc7 100755
--- a/config/plugins/interactive_environments/cellxgene/docker/run_cellxgene.sh
+++ b/config/plugins/interactive_environments/cellxgene/docker/run_cellxgene.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
 
-cellxgene launch --host 0.0.0.0 --port 80 /input/file.*
+cellxgene launch --host 0.0.0.0 --port 8080 /tmp/galaxy_cellxgene*

From 3c56b5917126dcdd63ad3219d0b8d8373ccedf73 Mon Sep 17 00:00:00 2001
From: Bjoern Gruening <bjoern.gruening@gmail.com>
Date: Sun, 5 May 2019 17:57:20 +0200
Subject: [PATCH 042/114] make entry_point_url parsing more robust

---
 lib/galaxy/webapps/galaxy/controllers/realtime.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index 895c04bda54..3864c725114 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -102,7 +102,7 @@ class RealTime(BaseUIController):
                     rval = '%s//%s.%s.%s.%s.%s/' % (trans.request.host_url.split('//', 1)[0], entry_point.__class__.__name__.lower(), trans.security.encode_id(entry_point.id),
                             entry_point.token, trans.app.config.realtime_prefix, trans.request.host)
                     if entry_point.entry_url:
-                        rval = '%s%s' % (rval, entry_point.entry_url)
+                        rval = '%s/%s' % (rval.rstrip('/'), entry_point.entry_url.lstrip('/'))
                     return trans.response.send_redirect(rval)
                 elif entry_point.deleted:
                     return trans.show_error_message('RealTimeTool has ended. You will have to start a new one.')

From ba6fc1981028b00f02298afc56fdf71bfd578387 Mon Sep 17 00:00:00 2001
From: Bjoern Gruening <bjoern.gruening@gmail.com>
Date: Sun, 5 May 2019 18:02:28 +0200
Subject: [PATCH 043/114] add initial condor runner support

---
 lib/galaxy/jobs/runners/condor.py | 110 +++++++++++++++++++++++++++++-
 1 file changed, 107 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/jobs/runners/condor.py b/lib/galaxy/jobs/runners/condor.py
index ed2d5446089..0ed48b59cda 100644
--- a/lib/galaxy/jobs/runners/condor.py
+++ b/lib/galaxy/jobs/runners/condor.py
@@ -12,6 +12,10 @@ it at this time.
 """
 import logging
 import os
+import shlex
+import subprocess
+import tempfile
+import time
 
 from galaxy import model
 from galaxy.jobs.runners import (
@@ -165,6 +169,9 @@ class CondorJobRunner(AsynchronousJobRunner):
         # Add to our 'queue' of jobs to monitor
         self.monitor_queue.put(cjs)
 
+        log.info('########### CondorExternalJobID befor handle_container %s' % external_job_id)
+        self.__handle_container(job_wrapper, external_job_id)
+
     def check_watched_items(self):
         """
         Called by the monitor thread to look at each watched job and deal
@@ -174,6 +181,7 @@ class CondorJobRunner(AsynchronousJobRunner):
         for cjs in self.watched:
             job_id = cjs.job_id
             galaxy_id_tag = cjs.job_wrapper.get_id_tag()
+            log.debug("### (%s/%s) whats-up" % (galaxy_id_tag, job_id))
             try:
                 if os.stat(cjs.user_log).st_size == cjs.user_log_size:
                     new_watched.append(cjs)
@@ -219,9 +227,43 @@ class CondorJobRunner(AsynchronousJobRunner):
         """Attempts to delete a job from the DRM queue"""
         job = job_wrapper.get_job()
         external_id = job.job_runner_external_id
-        failure_message = condor_stop(external_id)
-        if failure_message:
-            log.debug("(%s). Failed to stop condor %s" % (external_id, failure_message))
+        galaxy_id_tag = job_wrapper.get_id_tag()
+        if job.container:
+            try:
+                log.info("stop_job(): %s: trying to stop container .... (%s)" % (job.id, external_id))
+                #self.watched = [cjs for cjs in self.watched if cjs.job_id != external_id]
+                new_watch_list = list()
+                cjs = None
+                for tcjs in self.watched:
+                    if tcjs.job_id != external_id:
+                        new_watch_list.append(tcjs)
+                    else:
+                        cjs = tcjs
+                        break
+                self.watched = new_watch_list
+                self._stop_container(job_wrapper)
+                #self.watched.append(cjs)
+                if cjs.job_wrapper.get_state() != model.Job.states.DELETED:
+                    external_metadata = not asbool(cjs.job_wrapper.job_destination.params.get("embed_metadata_in_job", True))
+                    if external_metadata:
+                        self._handle_metadata_externally(cjs.job_wrapper, resolve_requirements=True)
+                    log.debug("(%s/%s) job has completed" % (galaxy_id_tag, external_id))
+                    self.work_queue.put((self.finish_job, cjs))
+                
+                
+            except Exception as e:
+                log.warning("stop_job(): %s: trying to stop container failed. (%s)" % (job.id, e))
+                try:
+                    self._kill_container(job_wrapper)
+                except Exception as e:
+                    log.warning("stop_job(): %s: trying to kill container failed. (%s)" % (job.id, e))
+                    failure_message = condor_stop(external_id)
+                    if failure_message:
+                        log.debug("(%s). Failed to stop condor %s" % (external_id, failure_message))
+        else:
+            failure_message = condor_stop(external_id)
+            if failure_message:
+                log.debug("(%s). Failed to stop condor %s" % (external_id, failure_message))
 
     def recover(self, job, job_wrapper):
         """Recovers jobs stuck in the queued/running state when Galaxy started"""
@@ -246,3 +288,65 @@ class CondorJobRunner(AsynchronousJobRunner):
             log.debug("(%s/%s) is still in DRM queued state, adding to the DRM queue" % (job.id, job.job_runner_external_id))
             cjs.running = False
             self.monitor_queue.put(cjs)
+
+
+
+    def _stop_container(self, job_wrapper):
+        return self._run_container_command(job_wrapper, 'stop')
+
+    def _kill_container(self, job_wrapper):
+        return self._run_container_command(job_wrapper, 'kill')
+
+    def _run_container_command(self, job_wrapper, command):
+        job = job_wrapper.get_job()
+        external_id = job.job_runner_external_id
+        if job:
+            cont = job.container
+            if cont:
+                if cont.container_type == 'docker':
+                    return self._run_command(cont.container_info['commands'][command], external_id)[0]
+
+    def _run_command(self, command, external_job_id):
+        command = 'condor_ssh_to_job %s %s' % (external_job_id, command)
+
+        p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, close_fds=True, preexec_fn=os.setpgrp)
+        stdout, stderr = p.communicate()
+        exit_code = p.returncode
+        ret = None
+        if exit_code == 0:
+            ret = stdout.strip()
+        else:
+            log.debug(stderr)
+        #exit_code = subprocess.call(command,
+        #                            shell=True,
+        #                            preexec_fn=os.setpgrp)
+        log.debug('_run_command(%s) exit code (%s) and failure: %s', command, exit_code, stderr)
+        return (exit_code, ret)
+
+    def __handle_container(self, job_wrapper, external_job_id):
+        job = job_wrapper.get_job()
+        if job:
+            cont = job.container
+            if cont:
+                if cont.container_type == 'docker':
+                    # Handle getting RealTimeTool ports, if applicable
+                    eps = job.realtimetool_entry_points
+                    if eps:
+                        max_command_attempts = 10 + 1
+                        ports_raw = None
+                        for i in range(1, max_command_attempts):
+                            with tempfile.TemporaryFile() as stdout_file:
+                                exit_code, ports_raw = self._run_command(cont.container_info['commands']['port'], external_job_id)
+                                if exit_code == 0:
+                                    break
+                            if i != max_command_attempts:
+                                t = 2 * i
+                                log.debug("Container not found during port check, sleeping for %s seconds.", t)
+                                time.sleep(t)
+                        log.warn('\n\n RAW ports: (%s)' % ports_raw)
+                        if ports_raw is not None:
+                            self.app.realtime_manager.configure_entry_points_raw_docker_ports(job, ports_raw)
+                        else:
+                            log.error('Unable to run port command (%s): %s', cont.container_info['commands']['port'], exit_code)
+
+

From 0a15a46086a576427da24c0bcf188e6cc43b8a05 Mon Sep 17 00:00:00 2001
From: Bjoern Gruening <bjoern.gruening@gmail.com>
Date: Sun, 5 May 2019 18:03:42 +0200
Subject: [PATCH 044/114] add initial test tools for a interactive Vis

---
 .../tools/interactive_tools_cellxgene.xml     | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 test/functional/tools/interactive_tools_cellxgene.xml

diff --git a/test/functional/tools/interactive_tools_cellxgene.xml b/test/functional/tools/interactive_tools_cellxgene.xml
new file mode 100644
index 00000000000..9369b99ef3e
--- /dev/null
+++ b/test/functional/tools/interactive_tools_cellxgene.xml
@@ -0,0 +1,23 @@
+<tool id="interactive_tool_cellxgene" tool_type="realtime" name="Interactive CellXgene Environment" version="0.1">
+    <requirements>
+        <container type="docker">quay.io/galaxy/cellxgene-galaxy-ie:ie2</container>
+    </requirements>
+    <ports>
+        <port name="cellxgene_single_cell_visualisation" url="0.0.0.0">8080</port>
+    </ports>
+    <command>
+        cp '$input' /tmp/galaxy_cellxgene_${input.element_identifier}.h5ad;
+        /run_cellxgene.sh
+    </command>
+    <inputs>
+        <param name="input" type="data" format="h5ad" label="Concatenate Dataset"/>
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="txt" />
+    </outputs>
+    <tests>
+    </tests>
+    <help>
+    Interactive tool for visualising AnnData.
+    </help>
+</tool>

From 67e8ad360a3f7b882eb1393d0a23c58437c8b4fa Mon Sep 17 00:00:00 2001
From: Bjoern Gruening <bjoern.gruening@gmail.com>
Date: Sun, 5 May 2019 18:05:41 +0200
Subject: [PATCH 045/114] add initial Jupyter notebook tool, this tool supports
 loading a notebook and execute it automatically, with that it can run in a
 workflow

---
 test/functional/tools/default_notebook.ipynb  | 53 +++++++++++++
 .../interactive_tool_juypter_notebook.xml     | 77 +++++++++++++++++++
 2 files changed, 130 insertions(+)
 create mode 100644 test/functional/tools/default_notebook.ipynb
 create mode 100644 test/functional/tools/interactive_tool_juypter_notebook.xml

diff --git a/test/functional/tools/default_notebook.ipynb b/test/functional/tools/default_notebook.ipynb
new file mode 100644
index 00000000000..e9573752478
--- /dev/null
+++ b/test/functional/tools/default_notebook.ipynb
@@ -0,0 +1,53 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Welcome to the interactive Galaxy IPython Notebook."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can access your data via the dataset number. Using a Python kernel, you can access dataset number 42 with ``handle = open(get(42), 'r')``.\n",
+    "To save data, write your data to a file, and then call ``put('filename.txt')``. The dataset will then be available in your galaxy history.\n<br>",
+    "When using a non-Python kernel, ``get`` and ``put`` are available as command-line tools, which can be accessed using system calls in R, Julia, and Ruby. For example, to read dataset number 42 into R, you can write ```handle <- file(system('get -i 42', intern = TRUE))```.\n",
+    "To save data in R, write the data to a file and then call ``system('put -p filename.txt')``.\n",
+    "Notebooks can be saved to Galaxy by clicking the large green button at the top right of the IPython interface.<br>\n",
+    "More help and informations can be found on the project [website](https://github.com/bgruening/docker-jupyter-notebook)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 2",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/test/functional/tools/interactive_tool_juypter_notebook.xml b/test/functional/tools/interactive_tool_juypter_notebook.xml
new file mode 100644
index 00000000000..e5c8d57921d
--- /dev/null
+++ b/test/functional/tools/interactive_tool_juypter_notebook.xml
@@ -0,0 +1,77 @@
+<tool id="interactive_tool_juypter_notebook" tool_type="realtime" name="Interactive Jupyter Notebook" version="0.1">
+    <requirements>
+        <container type="docker">bgruening/docker-jupyter-notebook:ie2</container>
+    </requirements>
+    <ports>
+        <port name="Jypter Interactive Environment" url_entry_point="ipython/tree">8888</port>
+    </ports>
+    <command detect_errors="aggressive"><![CDATA[
+        #import re
+        export GALAXY_WORKING_DIR=`pwd` &&
+        mkdir -p ./jupyter/outputs/ &&
+        mkdir -p ./jupyter/data &&
+
+        #for $i, $input in enumerate($inputs):
+            #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier))
+            ln -sf '$input' './jupyter/data/${cleaned_name}_${i}' &&
+        #end for
+
+        ## change into the directory where the notebooks are located
+        cd ./jupyter/ &&
+
+        #if $mode.mode_select == 'scratch':
+            ## copy default notebook
+            cp '$__tool_directory__/default_notebook.ipynb' ./ipython_galaxy_notebook.ipynb &&
+            /startup.sh
+        #else:
+
+            #for $i, $ipynb in enumerate($mode.ipynbs):
+                #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier))
+                cp '$ipynb' ./${cleaned_name}_${i}.ipynb &&
+            #end for
+            #if $mode.run_it:
+                jupyter nbconvert --to notebook --execute --output ./ipython_galaxy_notebook.ipynb --allow-errors  ./*.ipynb &&
+                mv ./ipython_galaxy_notebook.ipynb $jupyter_notebook
+            #else:
+                /startup.sh
+            #end if
+
+        #end if
+    ]]>
+    </command>
+    <inputs>
+
+        <conditional name="mode">
+            <param argument="mode_select" type="select" label="Do you already have a notebook?" help="If not, no problem we will provide you with a default one.">
+                <option value="scratch">Start with a fresh notebook</option>
+                <option value="previous">Load a previous notebook</option>
+            </param>
+            <when value="scratch"/>
+            <when value="previous">
+                <param name="ipynbs" type="data" format="ipynb" multiple="true" label="IPython Notebook"/>
+                <param name="run_it" type="boolean" truevalue="true" falsevalue="false" label="Execute notebook and return a new one."
+                    help="This option is useful in workflows when you just want to execute a notebook and not dive into the webfrontend."/>
+            </when>
+        </conditional>
+
+        <param name="inputs" type="data" optional="true" multiple="true" label="Include data into the environment"/>
+
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="txt">
+            <filter>(mode['mode_select'] == 'previous' and not mode['run_it']) or mode['mode_select'] == 'scratch'</filter>
+        </data>
+        <data name="jupyter_notebook" format="ipynb" label="Executed Notebook">
+            <filter>mode['mode_select'] == 'previous'</filter>
+            <filter>mode['run_it']</filter>
+        </data>
+        <!--data auto_format="true" name="collected_container_data">
+            <discover_datasets pattern="__designation__" directory="./jupyter/outputs/" visible="true" recurse="true" />
+        </data-->
+    </outputs>
+    <tests>
+    </tests>
+    <help>
+    Interactive tool.
+    </help>
+</tool>

From dd3839a75bfc057984c6fb0346924a0dc969480f Mon Sep 17 00:00:00 2001
From: Bjoern Gruening <bjoern.gruening@gmail.com>
Date: Sun, 5 May 2019 18:19:03 +0200
Subject: [PATCH 046/114] fix bug in setting the --net option during Docker
 startup

---
 lib/galaxy/tools/deps/container_classes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/tools/deps/container_classes.py b/lib/galaxy/tools/deps/container_classes.py
index 6594f92b564..17a01789c68 100644
--- a/lib/galaxy/tools/deps/container_classes.py
+++ b/lib/galaxy/tools/deps/container_classes.py
@@ -273,7 +273,7 @@ class DockerContainer(Container, HasDockerLikeVolumes):
             volumes_from=volumes_from,
             env_directives=env_directives,
             working_directory=working_directory,
-            net=self.prop("net", "none"),  # By default, docker instance has networking disabled
+            net=self.prop("net", None),  # By default, docker instance has networking disabled
             auto_rm=asbool(self.prop("auto_rm", docker_util.DEFAULT_AUTO_REMOVE)),
             set_user=self.prop("set_user", docker_util.DEFAULT_SET_USER),
             run_extra_arguments=self.prop("run_extra_arguments", docker_util.DEFAULT_RUN_EXTRA_ARGUMENTS),

From 09397f8f9603b10a2cc12488abb3aaaad1e6bfc8 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 13 May 2019 14:58:21 -0400
Subject: [PATCH 047/114] Update example RealTimeTools to use current syntax.

---
 ...ve_tools_cellxgene.xml => realtimetool_cellxgene.xml} | 8 +++++---
 ...er_notebook.xml => realtimetool_juypter_notebook.xml} | 9 ++++++---
 2 files changed, 11 insertions(+), 6 deletions(-)
 rename test/functional/tools/{interactive_tools_cellxgene.xml => realtimetool_cellxgene.xml} (81%)
 rename test/functional/tools/{interactive_tool_juypter_notebook.xml => realtimetool_juypter_notebook.xml} (94%)

diff --git a/test/functional/tools/interactive_tools_cellxgene.xml b/test/functional/tools/realtimetool_cellxgene.xml
similarity index 81%
rename from test/functional/tools/interactive_tools_cellxgene.xml
rename to test/functional/tools/realtimetool_cellxgene.xml
index 9369b99ef3e..67c08ce118b 100644
--- a/test/functional/tools/interactive_tools_cellxgene.xml
+++ b/test/functional/tools/realtimetool_cellxgene.xml
@@ -2,9 +2,11 @@
     <requirements>
         <container type="docker">quay.io/galaxy/cellxgene-galaxy-ie:ie2</container>
     </requirements>
-    <ports>
-        <port name="cellxgene_single_cell_visualisation" url="0.0.0.0">8080</port>
-    </ports>
+    <realtime>
+        <entry_point name="cellxgene_single_cell_visualisation">
+            <port>8080</port>
+        </entry_point>
+    </realtime>
     <command>
         cp '$input' /tmp/galaxy_cellxgene_${input.element_identifier}.h5ad;
         /run_cellxgene.sh
diff --git a/test/functional/tools/interactive_tool_juypter_notebook.xml b/test/functional/tools/realtimetool_juypter_notebook.xml
similarity index 94%
rename from test/functional/tools/interactive_tool_juypter_notebook.xml
rename to test/functional/tools/realtimetool_juypter_notebook.xml
index e5c8d57921d..f8c4f486b55 100644
--- a/test/functional/tools/interactive_tool_juypter_notebook.xml
+++ b/test/functional/tools/realtimetool_juypter_notebook.xml
@@ -2,9 +2,12 @@
     <requirements>
         <container type="docker">bgruening/docker-jupyter-notebook:ie2</container>
     </requirements>
-    <ports>
-        <port name="Jypter Interactive Environment" url_entry_point="ipython/tree">8888</port>
-    </ports>
+    <realtime>
+        <entry_point name="Jypter Interactive Environment">
+            <port>8888</port>
+            <url>ipython/tree</url>
+        </entry_point>
+    </realtime>
     <command detect_errors="aggressive"><![CDATA[
         #import re
         export GALAXY_WORKING_DIR=`pwd` &&

From 13c3f77188d6c32a250c29be4f95331797ba4d29 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 13 May 2019 15:13:11 -0400
Subject: [PATCH 048/114] flake8 for lib/galaxy/jobs/runners/condor.py

---
 lib/galaxy/jobs/runners/condor.py | 21 ++++++---------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/lib/galaxy/jobs/runners/condor.py b/lib/galaxy/jobs/runners/condor.py
index 0ed48b59cda..019ce72083f 100644
--- a/lib/galaxy/jobs/runners/condor.py
+++ b/lib/galaxy/jobs/runners/condor.py
@@ -12,9 +12,7 @@ it at this time.
 """
 import logging
 import os
-import shlex
 import subprocess
-import tempfile
 import time
 
 from galaxy import model
@@ -231,7 +229,7 @@ class CondorJobRunner(AsynchronousJobRunner):
         if job.container:
             try:
                 log.info("stop_job(): %s: trying to stop container .... (%s)" % (job.id, external_id))
-                #self.watched = [cjs for cjs in self.watched if cjs.job_id != external_id]
+                # self.watched = [cjs for cjs in self.watched if cjs.job_id != external_id]
                 new_watch_list = list()
                 cjs = None
                 for tcjs in self.watched:
@@ -242,15 +240,13 @@ class CondorJobRunner(AsynchronousJobRunner):
                         break
                 self.watched = new_watch_list
                 self._stop_container(job_wrapper)
-                #self.watched.append(cjs)
+                # self.watched.append(cjs)
                 if cjs.job_wrapper.get_state() != model.Job.states.DELETED:
                     external_metadata = not asbool(cjs.job_wrapper.job_destination.params.get("embed_metadata_in_job", True))
                     if external_metadata:
                         self._handle_metadata_externally(cjs.job_wrapper, resolve_requirements=True)
                     log.debug("(%s/%s) job has completed" % (galaxy_id_tag, external_id))
                     self.work_queue.put((self.finish_job, cjs))
-                
-                
             except Exception as e:
                 log.warning("stop_job(): %s: trying to stop container failed. (%s)" % (job.id, e))
                 try:
@@ -289,8 +285,6 @@ class CondorJobRunner(AsynchronousJobRunner):
             cjs.running = False
             self.monitor_queue.put(cjs)
 
-
-
     def _stop_container(self, job_wrapper):
         return self._run_container_command(job_wrapper, 'stop')
 
@@ -317,7 +311,7 @@ class CondorJobRunner(AsynchronousJobRunner):
             ret = stdout.strip()
         else:
             log.debug(stderr)
-        #exit_code = subprocess.call(command,
+        # exit_code = subprocess.call(command,
         #                            shell=True,
         #                            preexec_fn=os.setpgrp)
         log.debug('_run_command(%s) exit code (%s) and failure: %s', command, exit_code, stderr)
@@ -335,10 +329,9 @@ class CondorJobRunner(AsynchronousJobRunner):
                         max_command_attempts = 10 + 1
                         ports_raw = None
                         for i in range(1, max_command_attempts):
-                            with tempfile.TemporaryFile() as stdout_file:
-                                exit_code, ports_raw = self._run_command(cont.container_info['commands']['port'], external_job_id)
-                                if exit_code == 0:
-                                    break
+                            exit_code, ports_raw = self._run_command(cont.container_info['commands']['port'], external_job_id)
+                            if exit_code == 0:
+                                break
                             if i != max_command_attempts:
                                 t = 2 * i
                                 log.debug("Container not found during port check, sleeping for %s seconds.", t)
@@ -348,5 +341,3 @@ class CondorJobRunner(AsynchronousJobRunner):
                             self.app.realtime_manager.configure_entry_points_raw_docker_ports(job, ports_raw)
                         else:
                             log.error('Unable to run port command (%s): %s', cont.container_info['commands']['port'], exit_code)
-
-

From b7072d82baff9d1fbcea5c423613ebf2dbfcbf34 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 13 May 2019 15:34:47 -0400
Subject: [PATCH 049/114] mock

---
 test/unit/jobs/test_runner_local.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/unit/jobs/test_runner_local.py b/test/unit/jobs/test_runner_local.py
index acaafbc5557..81d19aa4493 100644
--- a/test/unit/jobs/test_runner_local.py
+++ b/test/unit/jobs/test_runner_local.py
@@ -144,6 +144,7 @@ class MockJobWrapper(object):
         self.shell = "/bin/bash"
         self.cleanup_job = "never"
         self.tmp_dir_creation_statement = ""
+        self.realtimetools = []
 
         # Cruft for setting metadata externally, axe at some point.
         self.external_output_metadata = bunch.Bunch(

From 2866b5ff1357e99ce4c984b9b546f337b0ae6307 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 13 May 2019 16:01:18 -0400
Subject: [PATCH 050/114] Update docs

---
 doc/source/admin/special_topics/realtimetools.rst | 6 +++---
 lib/galaxy/jobs/__init__.py                       | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/source/admin/special_topics/realtimetools.rst b/doc/source/admin/special_topics/realtimetools.rst
index 8c532764dba..3d64c206de1 100644
--- a/doc/source/admin/special_topics/realtimetools.rst
+++ b/doc/source/admin/special_topics/realtimetools.rst
@@ -5,7 +5,7 @@ A Galaxy RealTimeTool allows launching a container-backed Galaxy Tool
 and enabling a Galaxy User to gain access to content inside in real-time.
 
 
-How Galaxy RealTimeTools Work
+How Galaxy RealTimeTools work
 -----------------------------
 
 A RealTimeTool is defined in the same familiar way as standard Galaxy Tools,
@@ -39,8 +39,8 @@ Some important benefits of using Galaxy RealTimeTools
 
 
 
-Server configuration of Galaxy RealTimeTools
---------------------------------------------
+Server-side configuration of Galaxy RealTimeTools
+-------------------------------------------------
 
 The **galaxy.yml** file will need to be populated as seen in config/galaxy.yml.realtime.
 
diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py
index a3453123d9b..04867e2c9b3 100644
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -1046,7 +1046,7 @@ class JobWrapper(HasResourceParameters):
         # if the server was stopped and restarted before the job finished
         job.command_line = unicodify(self.command_line)
         job.dependencies = self.tool.dependencies
-        self.realtimetools = tool_evaluator.realtimetools
+        self.realtimetools = getattr(tool_evaluator, 'realtimetools', None)
         self.sa_session.add(job)
         self.sa_session.flush()
         # Return list of all extra files

From 2eb2106985125c88fa19ded6a5a40d8c30b64a78 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 13 May 2019 16:34:12 -0400
Subject: [PATCH 051/114] Ports to test.

---
 test/unit/tools/test_evaluation.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/unit/tools/test_evaluation.py b/test/unit/tools/test_evaluation.py
index f85ceb227ba..3f73a1131fd 100644
--- a/test/unit/tools/test_evaluation.py
+++ b/test/unit/tools/test_evaluation.py
@@ -270,6 +270,7 @@ class MockTool(object):
         self.app = app
         self.hooks_called = []
         self.environment_variables = []
+        self.ports = []
         self._config_files = []
         self._command_line = "bwa --thresh=$thresh --in=$input1 --out=$output1"
         self._params = {"thresh": self.test_thresh_param()}

From 6cc43da9b79a220ee479c2a4191afbb47eacaebf Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 13 May 2019 17:03:00 -0400
Subject: [PATCH 052/114] Doc formatting

---
 .../admin/special_topics/realtimetools.rst       | 16 ++++++++--------
 lib/galaxy/tools/evaluation.py                   |  4 +++-
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/doc/source/admin/special_topics/realtimetools.rst b/doc/source/admin/special_topics/realtimetools.rst
index 3d64c206de1..fd02ebae5a7 100644
--- a/doc/source/admin/special_topics/realtimetools.rst
+++ b/doc/source/admin/special_topics/realtimetools.rst
@@ -28,14 +28,14 @@ Note that name, port, and url are each able to be templated from the RealTimeToo
 Some important benefits of using Galaxy RealTimeTools
 -----------------------------------------------------
 
-- You can have and access any number of RealTimeTools at a time (admin configurable)
-- If you accidentally close the RealTimeTool browser window, you can regain access by selecting from a list of active RealTimeTools
-- A single RealTimeTool can grant access to multiple running applications, servers,  and interfaces
-- RealTimeTools can be added to Galaxy Workflows
-- Native, out-of-the box support for RealTimeTools in Galaxy via uWSGI proxying
-- RealTimeTools are bonafide Galaxy Tools; just specify tool_type as "realtime" and list the ports you want to expose 
-- RealTimeTools can be added to and installed from the ToolShed.
-- R Shiny apps, Javascript-based VNC access to desktop environments, genome-browsers-in-a-box, interactive notebook environments, etc, are all possible with RealTimeTools
+- You can have and access **any number of RealTimeTools at a time** (admin configurable)
+- If you accidentally close the **RealTimeTool browser window**, you can **regain access** by selecting from a **list of active RealTimeTools**
+- A single **RealTimeTool** can **grant access** to **multiple running applications, servers, and interfaces**
+- **RealTimeTools** can be **added to** Galaxy **Workflows**
+- **Native, out-of-the box support for RealTimeTools** in Galaxy via uWSGI proxying
+- **RealTimeTools** are **bonafide Galaxy Tools**; just specify tool_type as "realtime" and list the ports you want to expose
+- **RealTimeTools** can be **added** to and **installed from the ToolShed**.
+- **R Shiny apps**, **Javascript-based VNC** access to desktop environments, **genome-browsers-in-a-box**, **interactive notebook environments**, etc, are all possible with **RealTimeTools**
 
 
 
diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py
index 6aff14b7fb9..174ee2dff86 100644
--- a/lib/galaxy/tools/evaluation.py
+++ b/lib/galaxy/tools/evaluation.py
@@ -425,7 +425,9 @@ class ToolEvaluator(object):
                 ep_dict[key] = val
             rtt.append(ep_dict)
         self.realtimetools = rtt
-        self.app.realtime_manager.create_realtime(self.job, self.tool, rtt)
+        rtt_man = getattr(self.app, "realtime_manager", None)
+        if rtt_man:
+            rtt_man.create_realtime(self.job, self.tool, rtt)
         return rtt
 
     def __sanitize_param_dict(self, param_dict):

From 5ee6e65d12a2434a9cba60ac638206c348ec5951 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 14 May 2019 07:10:07 -0400
Subject: [PATCH 053/114] More doc info

---
 .../admin/special_topics/realtimetools.rst    | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/doc/source/admin/special_topics/realtimetools.rst b/doc/source/admin/special_topics/realtimetools.rst
index fd02ebae5a7..840466e39ce 100644
--- a/doc/source/admin/special_topics/realtimetools.rst
+++ b/doc/source/admin/special_topics/realtimetools.rst
@@ -47,6 +47,7 @@ The **galaxy.yml** file will need to be populated as seen in config/galaxy.yml.r
 In the **uwsgi:** section:
 
 .. code-block:: yaml
+
   http-raw-body: true
   # master: true
 
@@ -66,6 +67,9 @@ In the **galaxy:** section:
   realtime_prefix: realtime
 
 
+The admin should modify the **route-host**s and **realtime_prefix** to match their preferred configuration.
+
+
 An example **job_conf.xml** file as seen in config/galaxy.yml.realtime:
 
 .. code-block:: xml
@@ -91,3 +95,21 @@ An example **job_conf.xml** file as seen in config/galaxy.yml.realtime:
                 </destination>
             </destinations>
         </job_conf> 
+
+
+Alternatively to the local job runner, RealTimeTools have been enabled for the condor job runner, e.g.:
+
+.. code-block:: xml
+
+        <destination id="condor" runner="condor">
+            <param id="docker_enabled">true</param>
+            <param id="docker_sudo">false</param>
+        </destination>
+
+
+Two example test RealTimeTools have been defined, and can be added to the **tool_conf.xml**:
+
+.. code-block:: xml
+
+        <tool file="../test/functional/tools/realtimetool_juypter_notebook.xml" />
+        <tool file="../test/functional/tools/realtimetool_cellxgene.xml" />

From b0baaabf2c2eebdee8f5a2aaeb94f49eb3733a2b Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 14 May 2019 08:04:55 -0400
Subject: [PATCH 054/114] notes on resource consumption; and condor

---
 doc/source/admin/special_topics/realtimetools.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/source/admin/special_topics/realtimetools.rst b/doc/source/admin/special_topics/realtimetools.rst
index 840466e39ce..47dd36fff0c 100644
--- a/doc/source/admin/special_topics/realtimetools.rst
+++ b/doc/source/admin/special_topics/realtimetools.rst
@@ -107,9 +107,19 @@ Alternatively to the local job runner, RealTimeTools have been enabled for the c
         </destination>
 
 
+**Note on resource consumption:** Keep in mind that Distributed Resource Management (DRM) / cluster systems may have a maximum runtime configured for jobs. From the Galaxy point of view, such a container could run as long as the user desires, this may not be scalable and an admin may want to restrict the runtime of RealTimeTools (and jobs in general). However, if the job is killed by the DRM, the user is not informed beforehand and data in the container could be discarded.
+
 Two example test RealTimeTools have been defined, and can be added to the **tool_conf.xml**:
 
 .. code-block:: xml
 
         <tool file="../test/functional/tools/realtimetool_juypter_notebook.xml" />
         <tool file="../test/functional/tools/realtimetool_cellxgene.xml" />
+
+
+A few words on the condor integration
+-------------------------------------
+
+Galaxy needs to be able to stop a container gracefully. This is not a problem with the local job runner, where we assume that Docker is either running on the same host. However, if you are using production scale DRM, like condor, then your job is running
+somewhere on your cluster and you can not easily **docker stop** your container. For the condor integration we are using a great
+condor feature and commandline utility called **condor_ssh_to_job**. This tool (assuming your condor setup is configured correctly) will bring us directly to the host in question and we can execute the **docker stop** command. Galaxy will simply run **condor_ssh_to_job <condor_job_id> docker stop <container_name>** to stop the container gracefully.

From 1c12c745768cab57e99e496c896b918f04f93c6f Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 14 May 2019 23:23:25 -0400
Subject: [PATCH 055/114] update docs

---
 doc/source/admin/special_topics/realtimetools.rst | 10 +++++-----
 lib/galaxy/jobs/runners/__init__.py               |  2 +-
 lib/galaxy/tools/evaluation.py                    |  2 +-
 test/unit/jobs/test_runner_local.py               |  1 -
 test/unit/tools/test_evaluation.py                |  1 -
 5 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/doc/source/admin/special_topics/realtimetools.rst b/doc/source/admin/special_topics/realtimetools.rst
index 47dd36fff0c..61e3361eb5b 100644
--- a/doc/source/admin/special_topics/realtimetools.rst
+++ b/doc/source/admin/special_topics/realtimetools.rst
@@ -21,7 +21,7 @@ information:
             </entry_point>
         </realtime>
 
-Note that name, port, and url are each able to be templated from the RealTimeTool's parameter dictionary.
+**Note** that name, port, and url are each able to be templated from the RealTimeTool's parameter dictionary.
 
 
 
@@ -42,7 +42,7 @@ Some important benefits of using Galaxy RealTimeTools
 Server-side configuration of Galaxy RealTimeTools
 -------------------------------------------------
 
-The **galaxy.yml** file will need to be populated as seen in config/galaxy.yml.realtime.
+The **galaxy.yml** file will need to be populated as seen in **config/galaxy.yml.realtime**.
 
 In the **uwsgi:** section:
 
@@ -70,7 +70,7 @@ In the **galaxy:** section:
 The admin should modify the **route-host**s and **realtime_prefix** to match their preferred configuration.
 
 
-An example **job_conf.xml** file as seen in config/galaxy.yml.realtime:
+An example **job_conf.xml** file as seen in **config/galaxy.yml.realtime**:
 
 .. code-block:: xml
 
@@ -107,9 +107,9 @@ Alternatively to the local job runner, RealTimeTools have been enabled for the c
         </destination>
 
 
-**Note on resource consumption:** Keep in mind that Distributed Resource Management (DRM) / cluster systems may have a maximum runtime configured for jobs. From the Galaxy point of view, such a container could run as long as the user desires, this may not be scalable and an admin may want to restrict the runtime of RealTimeTools (and jobs in general). However, if the job is killed by the DRM, the user is not informed beforehand and data in the container could be discarded.
+**Note on resource consumption:** Keep in mind that Distributed Resource Management (DRM) / cluster systems may have a maximum runtime configured for jobs. From the Galaxy point of view, such a container could run as long as the user desires, this may not be advisable and an admin may want to restrict the runtime of RealTimeTools *(and jobs in general)*. However, if the job is killed by the DRM, the user is not informed beforehand and data in the container could be discarded.
 
-Two example test RealTimeTools have been defined, and can be added to the **tool_conf.xml**:
+Two **example test RealTimeTools** have been defined, and can be added to the **config/tool_conf.xml**:
 
 .. code-block:: xml
 
diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py
index b9d6259bdb7..6d506347047 100644
--- a/lib/galaxy/jobs/runners/__init__.py
+++ b/lib/galaxy/jobs/runners/__init__.py
@@ -417,7 +417,7 @@ class BaseJobRunner(object):
             compute_tmp_directory = job_wrapper.tmp_directory()
 
         tool = job_wrapper.tool
-        guest_ports = [ep.get('port') for ep in job_wrapper.realtimetools]
+        guest_ports = [ep.get('port') for ep in getattr(job_wrapper, 'realtimetools', [])]
         tool_info = ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through, guest_ports=guest_ports)
         job_info = JobInfo(
             compute_working_directory,
diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py
index 174ee2dff86..e90216968a0 100644
--- a/lib/galaxy/tools/evaluation.py
+++ b/lib/galaxy/tools/evaluation.py
@@ -411,7 +411,7 @@ class ToolEvaluator(object):
         Populate RealTimeTools templated values.
         """
         rtt = []
-        for ep in self.tool.ports:
+        for ep in getattr(self.tool, 'ports', []):
             ep_dict = {}
             for key in 'port', 'name', 'url':
                 val = ep.get(key, None)
diff --git a/test/unit/jobs/test_runner_local.py b/test/unit/jobs/test_runner_local.py
index 81d19aa4493..acaafbc5557 100644
--- a/test/unit/jobs/test_runner_local.py
+++ b/test/unit/jobs/test_runner_local.py
@@ -144,7 +144,6 @@ class MockJobWrapper(object):
         self.shell = "/bin/bash"
         self.cleanup_job = "never"
         self.tmp_dir_creation_statement = ""
-        self.realtimetools = []
 
         # Cruft for setting metadata externally, axe at some point.
         self.external_output_metadata = bunch.Bunch(
diff --git a/test/unit/tools/test_evaluation.py b/test/unit/tools/test_evaluation.py
index 3f73a1131fd..f85ceb227ba 100644
--- a/test/unit/tools/test_evaluation.py
+++ b/test/unit/tools/test_evaluation.py
@@ -270,7 +270,6 @@ class MockTool(object):
         self.app = app
         self.hooks_called = []
         self.environment_variables = []
-        self.ports = []
         self._config_files = []
         self._command_line = "bwa --thresh=$thresh --in=$input1 --out=$output1"
         self._params = {"thresh": self.test_thresh_param()}

From 138e90413a8597e934251095203c22c220613c31 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Wed, 15 May 2019 08:29:03 -0400
Subject: [PATCH 056/114] More doc formatting

---
 doc/source/admin/special_topics/realtimetools.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/source/admin/special_topics/realtimetools.rst b/doc/source/admin/special_topics/realtimetools.rst
index 61e3361eb5b..4834e7c096d 100644
--- a/doc/source/admin/special_topics/realtimetools.rst
+++ b/doc/source/admin/special_topics/realtimetools.rst
@@ -33,7 +33,7 @@ Some important benefits of using Galaxy RealTimeTools
 - A single **RealTimeTool** can **grant access** to **multiple running applications, servers, and interfaces**
 - **RealTimeTools** can be **added to** Galaxy **Workflows**
 - **Native, out-of-the box support for RealTimeTools** in Galaxy via uWSGI proxying
-- **RealTimeTools** are **bonafide Galaxy Tools**; just specify tool_type as "realtime" and list the ports you want to expose
+- **RealTimeTools** are **bonafide Galaxy Tools**; just specify **tool_type as "realtime"** and list the ports you want to expose
 - **RealTimeTools** can be **added** to and **installed from the ToolShed**.
 - **R Shiny apps**, **Javascript-based VNC** access to desktop environments, **genome-browsers-in-a-box**, **interactive notebook environments**, etc, are all possible with **RealTimeTools**
 

From 133d8b78205f2c7ad75fe2d28867b14f4accf148 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Wed, 15 May 2019 08:43:10 -0400
Subject: [PATCH 057/114] missing t

---
 test/functional/tools/realtimetool_cellxgene.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/functional/tools/realtimetool_cellxgene.xml b/test/functional/tools/realtimetool_cellxgene.xml
index 6431479dbe6..696fe634601 100644
--- a/test/functional/tools/realtimetool_cellxgene.xml
+++ b/test/functional/tools/realtimetool_cellxgene.xml
@@ -8,7 +8,7 @@
         </entry_point>
     </realtime>
     <command><![CDATA[
-        cp '${inpu}' '/tmp/galaxy_cellxgene_${input.element_identifier}.h5ad'
+        cp '${input}' '/tmp/galaxy_cellxgene_${input.element_identifier}.h5ad'
         &&
         cellxgene launch --host 0.0.0.0 --port 80 /tmp/galaxy_cellxgene*
     ]]>

From 87fed7e8eede87a6ab97e1f0862a9727d9a19880 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 20 May 2019 10:08:50 -0400
Subject: [PATCH 058/114] re.sub on cellxgene input name

---
 test/functional/tools/realtimetool_cellxgene.xml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/functional/tools/realtimetool_cellxgene.xml b/test/functional/tools/realtimetool_cellxgene.xml
index 696fe634601..43b56d24cb7 100644
--- a/test/functional/tools/realtimetool_cellxgene.xml
+++ b/test/functional/tools/realtimetool_cellxgene.xml
@@ -8,9 +8,11 @@
         </entry_point>
     </realtime>
     <command><![CDATA[
-        cp '${input}' '/tmp/galaxy_cellxgene_${input.element_identifier}.h5ad'
+        #import re
+        #set $fancy_name = '/tmp/galaxy_cellxgene_' + re.sub('[^\w\-_]', '_', $input.element_identifier) + '.h5ad'
+        cp '${input}' '${fancy_name}'
         &&
-        cellxgene launch --host 0.0.0.0 --port 80 /tmp/galaxy_cellxgene*
+        cellxgene launch --host 0.0.0.0 --port 80 '${fancy_name}'
     ]]>
     </command>
     <inputs>

From 9d93823d50cd554ef7e50517d5d5d1129143fc29 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Mon, 20 May 2019 09:42:08 -0400
Subject: [PATCH 059/114] Simplify set_name stuff in container.

---
 lib/galaxy/jobs/runners/__init__.py        | 3 +--
 lib/galaxy/tools/deps/container_classes.py | 6 +-----
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py
index 6d506347047..5cbcb396724 100644
--- a/lib/galaxy/jobs/runners/__init__.py
+++ b/lib/galaxy/jobs/runners/__init__.py
@@ -433,8 +433,7 @@ class BaseJobRunner(object):
             destination_info,
             job_info
         )
-        if container:
-            container.set_name(job_wrapper=job_wrapper)
+        job_wrapper.set_container(container)
         return container
 
     def _handle_runner_state(self, runner_state, job_state):
diff --git a/lib/galaxy/tools/deps/container_classes.py b/lib/galaxy/tools/deps/container_classes.py
index 17a01789c68..899175e6492 100644
--- a/lib/galaxy/tools/deps/container_classes.py
+++ b/lib/galaxy/tools/deps/container_classes.py
@@ -68,7 +68,7 @@ class Container(object):
         self.destination_info = destination_info
         self.job_info = job_info
         self.container_description = container_description
-        self.container_name = container_name
+        self.container_name = container_name or uuid4().get_hex()
         self.container_info = {}
 
     def prop(self, name, default):
@@ -91,10 +91,6 @@ class Container(object):
         input command in container.
         """
 
-    def set_name(self, job_wrapper=None):
-        self.container_name = uuid4().get_hex()
-        job_wrapper.set_container(self)
-
 
 def preprocess_volumes(volumes_raw_str, container_type):
     """Process Galaxy volume specification string to either Docker or Singularity specification.

From 9136e2946da61dcf4288d0441c350f29a5e1371c Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 20 May 2019 10:47:36 -0400
Subject: [PATCH 060/114] Only set container if container exists, fixes tests
 failing due to Mock object.

---
 lib/galaxy/jobs/runners/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py
index 5cbcb396724..b9dc17e8e6c 100644
--- a/lib/galaxy/jobs/runners/__init__.py
+++ b/lib/galaxy/jobs/runners/__init__.py
@@ -433,7 +433,8 @@ class BaseJobRunner(object):
             destination_info,
             job_info
         )
-        job_wrapper.set_container(container)
+        if container:
+            job_wrapper.set_container(container)
         return container
 
     def _handle_runner_state(self, runner_state, job_state):

From 5cc0d8070d77d8f8c3f25a985261768931fb14e1 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 21 May 2019 08:24:59 -0400
Subject: [PATCH 061/114] Fix tool_util import

stupid .pyc files
---
 lib/galaxy/managers/realtime.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index 38ec7dfb62b..09aa2b2674a 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -6,7 +6,7 @@ from six import string_types
 from sqlalchemy import or_
 
 
-from galaxy.tools.deps.docker_util import parse_port_text as docker_parse_port_text
+from galaxy.tool_util.deps.docker_util import parse_port_text as docker_parse_port_text
 from galaxy.util.filelock import FileLock
 
 

From 6e01ca8070d2f7e14432f4ab43fa1fd2c908b7a4 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 21 May 2019 16:56:46 -0400
Subject: [PATCH 062/114] Change default realtime prefix to something that
 isn't broken.

---
 lib/galaxy/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py
index 2faad38e5e5..3941bf7d879 100644
--- a/lib/galaxy/config.py
+++ b/lib/galaxy/config.py
@@ -707,7 +707,7 @@ class Configuration(object):
         self.dynamic_proxy_golang_docker_address = kwargs.get("dynamic_proxy_golang_docker_address", "unix:///var/run/docker.sock")
         self.dynamic_proxy_golang_api_key = kwargs.get("dynamic_proxy_golang_api_key", None)
 
-        self.realtime_prefix = kwargs.get("realtime_prefix", "/realtime-proxy/access")
+        self.realtime_prefix = kwargs.get("realtime_prefix", "realtime")
 
         # Default chunk size for chunkable datatypes -- 64k
         self.display_chunk_size = int(kwargs.get('display_chunk_size', 65536))

From 58cbb50819fac3776e673a5f48d418d7ec0af1d4 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Mon, 20 May 2019 16:56:03 -0400
Subject: [PATCH 063/114] Refactor realtime entry point handling toward API
 usage.

- Add API entry point for fetching a target from an encoded entry point ID.
- Refactor target generation into a manager.
- Use galaxy.exceptions and exception handling instead of trans.show_error_message directly.
- Extend grid view to leverage this new API and use single-page error handling.
---
 .../scripts/entry/analysis/AnalysisRouter.js  |  3 +-
 client/galaxy/scripts/mvc/entrypoints/view.js | 27 +++++++++
 lib/galaxy/managers/realtime.py               | 20 +++++++
 .../webapps/galaxy/api/tool_entry_points.py   | 22 ++++++++
 lib/galaxy/webapps/galaxy/buildapp.py         |  2 +
 .../webapps/galaxy/controllers/realtime.py    | 56 ++++++-------------
 6 files changed, 91 insertions(+), 39 deletions(-)
 create mode 100644 client/galaxy/scripts/mvc/entrypoints/view.js
 create mode 100644 lib/galaxy/webapps/galaxy/api/tool_entry_points.py

diff --git a/client/galaxy/scripts/entry/analysis/AnalysisRouter.js b/client/galaxy/scripts/entry/analysis/AnalysisRouter.js
index 22fedba2871..481402d7d2f 100644
--- a/client/galaxy/scripts/entry/analysis/AnalysisRouter.js
+++ b/client/galaxy/scripts/entry/analysis/AnalysisRouter.js
@@ -22,6 +22,7 @@ import UserPreferences from "mvc/user/user-preferences";
 import CustomBuilds from "mvc/user/user-custom-builds";
 import Tours from "mvc/tours";
 import GridView from "mvc/grid/grid-view";
+import EntryPointGridView from "mvc/entrypoints/view";
 import GridShared from "mvc/grid/grid-shared";
 import Workflows from "mvc/workflow/workflow";
 import WorkflowImport from "components/WorkflowImport.vue";
@@ -115,7 +116,7 @@ export const getAnalysisRouter = Galaxy =>
 
         show_realtime_list: function() {
             this.page.display(
-                new GridView({
+                new EntryPointGridView({
                     url_base: `${getAppRoot()}realtime/list`,
                     active_tab: "analysis"
                 })
diff --git a/client/galaxy/scripts/mvc/entrypoints/view.js b/client/galaxy/scripts/mvc/entrypoints/view.js
new file mode 100644
index 00000000000..63789baba71
--- /dev/null
+++ b/client/galaxy/scripts/mvc/entrypoints/view.js
@@ -0,0 +1,27 @@
+import $ from "jquery";
+import axios from "axios";
+import GridView from "mvc/grid/grid-view";
+import { getAppRoot } from "onload";
+import { Toast } from "ui/toast";
+
+export default GridView.extend({
+    init_grid_elements: function() {
+        GridView.prototype.init_grid_elements.call(this);
+        $(".entry-point-link").click(event => {
+            const $link = $(event.target);
+            const entry_point_id = $link.attr("entry_point_id");
+            const url = `${getAppRoot()}api/entry_points/${entry_point_id}/access`;
+            const handlerError = err => {
+                let message = "Problem finding entry point target, please contact an admin.";
+                if (err.response) {
+                    message = err.response.data.err_msg;
+                }
+                Toast.error(message);
+            };
+            axios
+                .get(url)
+                .then(response => (window.location = response.data.target))
+                .catch(handlerError);
+        });
+    }
+});
diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index 09aa2b2674a..389019e38fd 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -6,6 +6,10 @@ from six import string_types
 from sqlalchemy import or_
 
 
+from galaxy import (
+    model,
+    exceptions
+)
 from galaxy.tool_util.deps.docker_util import parse_port_text as docker_parse_port_text
 from galaxy.util.filelock import FileLock
 
@@ -145,6 +149,7 @@ class RealTimeManager(object):
     """
 
     def __init__(self, app):
+        self.app = app
         self.model = app.model
         self.security = app.security
         self.sa_session = app.model.context
@@ -273,3 +278,18 @@ class RealTimeManager(object):
         if flush:
             self.sa_session.flush()
         self.propagator.remove_entry_point(entry_point)
+
+    def access_entry_point_target(self, trans, entry_point_id):
+        entry_point = trans.sa_session.query(model.RealTimeToolEntryPoint).get(entry_point_id)
+        if self.app.realtime_manager.can_access_entry_point(trans, entry_point):
+            if entry_point.active:
+                rval = '%s//%s.%s.%s.%s.%s/' % (trans.request.host_url.split('//', 1)[0], entry_point.__class__.__name__.lower(), trans.security.encode_id(entry_point.id),
+                        entry_point.token, self.app.config.realtime_prefix, trans.request.host)
+                if entry_point.entry_url:
+                    rval = '%s/%s' % (rval.rstrip('/'), entry_point.entry_url.lstrip('/'))
+                return rval
+            elif entry_point.deleted:
+                raise exceptions.MessageException('RealTimeTool has ended. You will have to start a new one.')
+            else:
+                raise exceptions.MessageException('RealTimeTool is not active. If you recently launched this tool it may not be ready yet, please wait a moment and refresh this page.')
+        raise exceptions.ItemAccessibilityException("You do not have access to this RealTimeTool entry point.")
diff --git a/lib/galaxy/webapps/galaxy/api/tool_entry_points.py b/lib/galaxy/webapps/galaxy/api/tool_entry_points.py
new file mode 100644
index 00000000000..5c22d64d15d
--- /dev/null
+++ b/lib/galaxy/webapps/galaxy/api/tool_entry_points.py
@@ -0,0 +1,22 @@
+""" API for asynchronous job running mechanisms can use to fetch or put files
+related to running and queued jobs.
+"""
+import logging
+
+from galaxy import exceptions
+from galaxy.managers.realtime import RealTimeManager
+from galaxy.web import expose_api_anonymous_and_sessionless
+from galaxy.web.base.controller import BaseAPIController
+
+log = logging.getLogger(__name__)
+
+
+class ToolEntryPointsAPIController(BaseAPIController):
+
+    @expose_api_anonymous_and_sessionless
+    def access_entry_point(self, trans, id, **kwd):
+        # Because of auto id encoding needed for link from grid, the item.id keyword must be 'id'
+        if not id:
+            raise exceptions.RequestParameterMissingException("Must supply entry point ID.")
+        entry_point_id = self.decode_id(id)
+        return {"target": RealTimeManager(self.app).access_entry_point_target(trans, entry_point_id)}
diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py
index 35672b6ff26..ae81d132595 100644
--- a/lib/galaxy/webapps/galaxy/buildapp.py
+++ b/lib/galaxy/webapps/galaxy/buildapp.py
@@ -359,6 +359,8 @@ def populate_api_routes(webapp, app):
     webapp.mapper.resource('tool', 'tools', path_prefix='/api')
     webapp.mapper.resource('dynamic_tools', 'dynamic_tools', path_prefix='/api')
 
+    webapp.mapper.connect('/api/entry_points/{id:.+?}/access', action='access_entry_point', controller="tool_entry_points")
+
     webapp.mapper.connect('/api/dependency_resolvers/clean', action="clean", controller="tool_dependencies", conditions=dict(method=["POST"]))
     webapp.mapper.connect('/api/dependency_resolvers/dependency', action="manager_dependency", controller="tool_dependencies", conditions=dict(method=["GET"]))
     webapp.mapper.connect('/api/dependency_resolvers/dependency', action="install_dependency", controller="tool_dependencies", conditions=dict(method=["POST"]))
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index 3864c725114..ee7abaf7763 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -4,9 +4,11 @@ Provides web interaction with RealTimeTools
 import logging
 
 from galaxy import (
+    exceptions,
     model,
     web
 )
+from galaxy.managers.realtime import RealTimeManager
 from galaxy.web import url_for
 from galaxy.web.base.controller import (
     BaseUIController,
@@ -24,6 +26,11 @@ class JobStatusColumn(grids.StateColumn):
         return super(JobStatusColumn, self).get_value(trans, grid, item.job)
 
 
+class EntryPointLinkColumn(grids.GridColumn):
+    def get_value(self, trans, grid, item):
+        return '<a class="entry-point-link" entry_point_id="%s">%s</div>' % (trans.security.encode_id(item.id), item.name)
+
+
 class RealTimeToolEntryPointListGrid(grids.Grid):
     def get_url_args(item):
         """
@@ -45,7 +52,7 @@ class RealTimeToolEntryPointListGrid(grids.Grid):
     default_filter = {"name": "All"}
     default_sort_key = "-update_time"
     columns = [
-        grids.TextColumn("Name", key="name", filterable="advanced", link=get_url_args),
+        EntryPointLinkColumn("Name", filterable="advanced"),
         grids.GridColumn("Active", key="active"),
         JobStatusColumn("Job Info", key="job_state", model_class=model.Job),
         grids.GridColumn("Created", key="created_time", format=time_ago),
@@ -71,44 +78,17 @@ class RealTime(BaseUIController):
     entry_point_grid = RealTimeToolEntryPointListGrid()
 
     @web.expose
-    def index(self, trans, entry_point_id=None, job_id=None, **kwd):
-        eps = []
-        if entry_point_id:
-            if not isinstance(entry_point_id, list):
-                entry_point_id = [entry_point_id]
-            eps = [trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).get(self.decode_id(ep_id)) for ep_id in entry_point_id]
-        if job_id:
-            job = trans.sa_session.query(trans.app.model.Job).get(self.decode_id(job_id))
-            for ep in job.realtimetool_entry_points:
-                eps.append(ep)
-        if eps:
-            if trans.app.realtime_manager.can_access_entry_points(trans, eps):
-                if len(eps) == 1:
-                    return self.access_entry_point(trans, id=trans.security.encode_id(eps[0].id))
-                else:
-                    return trans.response.send_redirect(url_for(controller="realtime", action="list"))
+    def index(self, trans, job_id=None, **kwd):
+        job = trans.sa_session.query(trans.app.model.Job).get(self.decode_id(job_id))
+        eps = job.realtimetool_entry_points
+        try:
+            if eps and len(eps) == 1:
+                redirect_target = RealTimeManager(self.app).access_entry_point_target(trans, eps[0].id)
             else:
-                return trans.show_error_message('Access not authorized.')
-        return trans.show_error_message('RealTimeTool instance not found')
-
-    @web.expose
-    def access_entry_point(self, trans, id=None):
-        # Because of auto id encoding needed for link from grid, the item.id keyword must be 'id'
-        if id:
-            entry_point_id = self.decode_id(id)
-            entry_point = trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).get(entry_point_id)
-            if trans.app.realtime_manager.can_access_entry_point(trans, entry_point):
-                if entry_point.active:
-                    rval = '%s//%s.%s.%s.%s.%s/' % (trans.request.host_url.split('//', 1)[0], entry_point.__class__.__name__.lower(), trans.security.encode_id(entry_point.id),
-                            entry_point.token, trans.app.config.realtime_prefix, trans.request.host)
-                    if entry_point.entry_url:
-                        rval = '%s/%s' % (rval.rstrip('/'), entry_point.entry_url.lstrip('/'))
-                    return trans.response.send_redirect(rval)
-                elif entry_point.deleted:
-                    return trans.show_error_message('RealTimeTool has ended. You will have to start a new one.')
-                else:
-                    return trans.show_error_message('RealTimeTool is not active. If you recently launched this tool it may not be ready yet, please wait a moment and refresh this page.')
-        return trans.show_error_message('Access not authorized.')
+                redirect_target = url_for(controller="realtime", action="list")
+            return trans.response.send_redirect(redirect_target)
+        except exceptions.MessageException as e:
+            return trans.show_error_message(str(e))
 
     @web.expose_api_anonymous
     def list(self, trans, **kwargs):

From 821778f783bbb9d7b56c8cb27b171f1c52601a18 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 21 May 2019 11:32:16 -0400
Subject: [PATCH 064/114] Move computing result_view for RTT to client.

---
 .../ToolEntryPoints/ToolEntryPoints.vue       | 83 +++++++++++++++++++
 client/galaxy/scripts/mvc/tool/tool-form.js   | 37 ++++-----
 lib/galaxy/managers/realtime.py               |  4 +-
 lib/galaxy/model/__init__.py                  | 11 +--
 lib/galaxy/tools/__init__.py                  | 14 +---
 lib/galaxy/tools/execute.py                   |  9 --
 .../webapps/galaxy/api/tool_entry_points.py   | 41 ++++++++-
 lib/galaxy/webapps/galaxy/api/tools.py        |  3 +-
 lib/galaxy/webapps/galaxy/buildapp.py         |  1 +
 .../webapps/galaxy/controllers/realtime.py    | 13 ---
 10 files changed, 151 insertions(+), 65 deletions(-)
 create mode 100644 client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue

diff --git a/client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue b/client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue
new file mode 100644
index 00000000000..2465785f2c7
--- /dev/null
+++ b/client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue
@@ -0,0 +1,83 @@
+<template>
+    <div class="infomessagelarge">
+        <p v-if="entryPoints.length==0" >
+            Waiting for RealTimeTool result view(s) to become available.
+        </p>
+        <p v-else-if="entryPoints.length==1" >
+            <span v-if="entryPoints[0].active" >
+                There is a RealTimeTool result view available, click here to display.
+            </span>
+            <span v-else>
+                There is a RealTimeTool result view available, waiting for view to become active...
+            </span>
+        </p>
+        <p v-else>
+            There is are multiple RealTimeTool result views available:
+            <ul>
+                <li v-for="entryPoint of entryPoints" v-bind:key="entryPoint.id" >
+                    {{ entryPoint.name }}
+                    <span v-if="entryPoint.active">
+                        (active)
+                    </span>
+                    <span v-else>
+                        (waiting to become active...)
+                    </span>
+                </li>
+            </ul>
+        </p>
+
+        You may also access all active RealTimeTools from the User menu.
+    </div>
+</template>
+
+<script>
+import { getAppRoot } from "onload/loadConfig";
+import axios from "axios";
+export default {
+    props: {
+        jobId: {
+            type: String,
+            required: true
+        }
+    },
+    data() {
+        return {
+            entryPoints: [],
+            interval: null,
+        };
+    },
+    created: function() {
+        this.reloadEntryPoints();
+    },
+    beforeDestroy: function(){
+        clearInterval(this.interval);
+    },
+    methods: {
+        reloadEntryPoints: function() {
+            clearInterval(this.interval);
+            const url = getAppRoot() + `api/entry_points?job_id=${this.jobId}`;
+            axios
+                .get(url)
+                .then(response => {
+                    const entryPoints = [];
+                    let allReady = true;
+                    response.data.forEach((entryPoint, i) => {
+                        entryPoints.push(entryPoint);
+                        if(! entryPoint.active) {
+                            allReady = false;
+                        }
+                    });
+                    this.entryPoints = entryPoints;
+                    if(! allReady || entryPoints.length == 0) {
+                        this.interval = setInterval(() => {
+                            this.reloadEntryPoints();
+                        }, 5000);
+                    }
+                })
+                .catch(e => {
+                    console.error(e);
+                });
+        }
+    }
+};
+</script>
\ No newline at end of file
diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js
index 94340e3382d..ab479f03a57 100644
--- a/client/galaxy/scripts/mvc/tool/tool-form.js
+++ b/client/galaxy/scripts/mvc/tool/tool-form.js
@@ -11,6 +11,8 @@ import Ui from "mvc/ui/ui-misc";
 import Modal from "mvc/ui/ui-modal";
 import ToolFormBase from "mvc/tool/tool-form-base";
 import Webhooks from "mvc/webhooks";
+import Vue from "Vue";
+import ToolEntryPoints from "components/ToolEntryPoints/ToolEntryPoints";
 
 var View = Backbone.View.extend({
     initialize: function(options) {
@@ -233,6 +235,19 @@ var View = Backbone.View.extend({
             success: function(response) {
                 callback && callback();
                 self.$el.children().hide();
+                if (response.produces_entry_points) {
+                    for (let job of response.jobs) {
+                        let toolEntryPointsInstance = Vue.extend(ToolEntryPoints);
+                        let vm = document.createElement("div");
+                        self.$el.append(vm);
+                        let instance = new toolEntryPointsInstance({
+                            propsData: {
+                                jobId: job.id
+                            }
+                        });
+                        instance.$mount(vm);
+                    }
+                }
                 self.$el.append(self._templateSuccess(response, job_def));
                 // Show Webhook if job is running
                 if (response.jobs && response.jobs.length > 0) {
@@ -347,24 +362,6 @@ var View = Backbone.View.extend({
         return inputs;
     },
 
-    _getViewResult: function(response) {
-        var vr = "";
-        if (response.view_result) {
-            for (const i in response.view_result) {
-                vr += `<div class="infomessagelarge">
-                            <p>
-                                <a href="${response.view_result[i]}">
-                                There is a RealTimeTool result view available, click here to display.</a>
-                            </p>
-                            <p>
-                                You may also access all active RealTimeTools from the User menu.
-                            </p>
-                        </div>`;
-            }
-        }
-        return vr;
-    },
-
     _templateRow: function(list, title, max = 3) {
         var blurb = "";
         list.sort(function(a, b) {
@@ -395,9 +392,7 @@ var View = Backbone.View.extend({
             var ninputsText = ninputs > 1 ? `${ninputs} inputs` : `this input`;
             var noutputsText = noutputs > 1 ? `${noutputs} outputs` : `this output`;
             var tool_name = this.form.model.get("name");
-            var view_result = this._getViewResult(response);
-            return `${view_result}
-                    <div class="donemessagelarge">
+            return `<div class="donemessagelarge">
                         <p>
                             Executed <b>${tool_name}</b> and successfully added ${njobsText} to the queue.
                         </p>
diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index 389019e38fd..81bfdf44e31 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -283,8 +283,10 @@ class RealTimeManager(object):
         entry_point = trans.sa_session.query(model.RealTimeToolEntryPoint).get(entry_point_id)
         if self.app.realtime_manager.can_access_entry_point(trans, entry_point):
             if entry_point.active:
+                request_host = trans.request.host
+                log.info("building access for host %s" % request_host)
                 rval = '%s//%s.%s.%s.%s.%s/' % (trans.request.host_url.split('//', 1)[0], entry_point.__class__.__name__.lower(), trans.security.encode_id(entry_point.id),
-                        entry_point.token, self.app.config.realtime_prefix, trans.request.host)
+                        entry_point.token, self.app.config.realtime_prefix, request_host)
                 if entry_point.entry_url:
                     rval = '%s/%s' % (rval.rstrip('/'), entry_point.entry_url.lstrip('/'))
                 return rval
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 8621596decf..b4399cfcb6c 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -1463,7 +1463,10 @@ class JobContainerAssociation(RepresentById):
         self.container_info = container_info or {}
 
 
-class RealTimeToolEntryPoint(RepresentById):
+class RealTimeToolEntryPoint(Dictifiable, RepresentById):
+    dict_collection_visible_keys = ['id', 'name', 'active']
+    dict_element_visible_keys = ['id', 'name', 'active']
+
     def __init__(self, job=None, name=None, token=None, tool_port=None, host=None, port=None, protocol=None,
                  entry_url=None, info=None, configured=False, deleted=False):
         self.job = job
@@ -1480,12 +1483,6 @@ class RealTimeToolEntryPoint(RepresentById):
         self.configured = configured
         self.deleted = deleted
 
-    def to_dict(self, *args, **kwds):
-        rval = dict(realtime_id=self.realtime.id)
-        for val in ['id', 'name', 'token', 'tool_port', 'entry_url', 'host', 'port', 'protocol', 'configured', 'created_time', 'modified_time', 'active']:
-            rval[val] = getattr(self, val)
-        return rval
-
     @property
     def active(self):
         if self.configured and not self.deleted:
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 6543d5da9c4..ac9bfd63ce4 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -411,6 +411,7 @@ class Tool(Dictifiable):
 
     tool_type = 'default'
     requires_setting_metadata = True
+    produces_entry_points = False
     default_tool_action = DefaultToolAction
     dict_collection_visible_keys = ['id', 'name', 'version', 'description', 'labels']
 
@@ -1464,8 +1465,7 @@ class Tool(Dictifiable):
                         job_errors=execution_tracker.execution_errors,
                         jobs=execution_tracker.successful_jobs,
                         output_collections=execution_tracker.output_collections,
-                        implicit_collections=execution_tracker.implicit_collections,
-                        view_result=execution_tracker.view_result)
+                        implicit_collections=execution_tracker.implicit_collections)
 
     def handle_single_execution(self, trans, rerun_remap_job_id, execution_slice, history, execution_cache=None, completed_job=None, collection_info=None):
         """
@@ -2054,10 +2054,6 @@ class Tool(Dictifiable):
             else:
                 group_inputs[input_index] = tool_dict
 
-    def get_view_result(self, job=None, dataset=None, **kwd):
-        """Returns a special view for a tool and a set of provided objects."""
-        return None
-
     def _get_job_remap(self, job):
         if job:
             if job.state == job.states.ERROR:
@@ -2443,6 +2439,7 @@ class ImportHistoryTool(Tool):
 
 class RealTimeTool(Tool):
     tool_type = 'realtime'
+    produces_entry_points = True
 
     def __remove_realtime_by_job(self, job):
         if job:
@@ -2462,11 +2459,6 @@ class RealTimeTool(Tool):
         job = job_wrapper.sa_session.query(model.Job).get(job_wrapper.job_id)
         self.__remove_realtime_by_job(job)
 
-    def get_view_result(self, job=None):
-        if job:
-            return self.app.url_for(controller='realtime', action='index', job_id=self.app.security.encode_id(job.id))
-        return None
-
 
 class DataManagerTool(OutputParameterJSONTool):
     tool_type = 'manage_data'
diff --git a/lib/galaxy/tools/execute.py b/lib/galaxy/tools/execute.py
index 66b09cde077..e5a10d7c18e 100644
--- a/lib/galaxy/tools/execute.py
+++ b/lib/galaxy/tools/execute.py
@@ -358,15 +358,6 @@ class ExecutionTracker(object):
             job_assoc.job_id = job.id
             self.trans.sa_session.add(job_assoc)
 
-    @property
-    def view_result(self):
-        rval = []
-        for job in self.successful_jobs:
-            result = self.tool.get_view_result(job=job)
-            if result:
-                rval.append(result)
-        return rval
-
 
 # Seperate these because workflows need to track their jobs belong to the invocation
 # in the database immediately and they can be recovered.
diff --git a/lib/galaxy/webapps/galaxy/api/tool_entry_points.py b/lib/galaxy/webapps/galaxy/api/tool_entry_points.py
index 5c22d64d15d..4ba72dde997 100644
--- a/lib/galaxy/webapps/galaxy/api/tool_entry_points.py
+++ b/lib/galaxy/webapps/galaxy/api/tool_entry_points.py
@@ -5,6 +5,7 @@ import logging
 
 from galaxy import exceptions
 from galaxy.managers.realtime import RealTimeManager
+from galaxy.util.dictifiable import dict_for
 from galaxy.web import expose_api_anonymous_and_sessionless
 from galaxy.web.base.controller import BaseAPIController
 
@@ -13,10 +14,48 @@ log = logging.getLogger(__name__)
 
 class ToolEntryPointsAPIController(BaseAPIController):
 
+    def __init__(self, app):
+        self.app = app
+        self.realtime_manager = RealTimeManager(app)
+
+    @expose_api_anonymous_and_sessionless
+    def index(self, trans, job_id=None, **kwd):
+        """
+        * GET /api/entry_points
+            Returns tool entry point information. Currently passing a job_id
+            parameter is required, as this becomes more general that won't be
+            needed.
+
+        :type   job_id: string
+        :param  job_id: Encoded job id
+
+        :rtype:     list
+        :returns:   list of entry point dictionaries.
+        """
+        job = trans.sa_session.query(trans.app.model.Job).get(self.decode_id(job_id))
+        if not self.realtime_manager.can_access_job(trans, job):
+            raise exceptions.ItemAccessibilityException()
+        entry_points = job.realtimetool_entry_points
+
+        rval = []
+        for entry_point in entry_points:
+            rval.append(self.encode_all_ids(trans, entry_point.to_dict(), True))
+        return rval
+
     @expose_api_anonymous_and_sessionless
     def access_entry_point(self, trans, id, **kwd):
+        """
+        * GET /api/entry_points/{id}/access
+            Return the URL target described by the entry point.
+
+        :type   id: string
+        :param  id: Encoded entry point id
+
+        :rtype:     dictionary
+        :returns:   dictionary containing target for realtime entry point
+        """
         # Because of auto id encoding needed for link from grid, the item.id keyword must be 'id'
         if not id:
             raise exceptions.RequestParameterMissingException("Must supply entry point ID.")
         entry_point_id = self.decode_id(id)
-        return {"target": RealTimeManager(self.app).access_entry_point_target(trans, entry_point_id)}
+        return {"target": self.realtime_manager.access_entry_point_target(trans, entry_point_id)}
diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py
index e3a0a4c0c58..64f9852c795 100644
--- a/lib/galaxy/webapps/galaxy/api/tools.py
+++ b/lib/galaxy/webapps/galaxy/api/tools.py
@@ -526,7 +526,7 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
         # TODO: check for errors and ensure that output dataset(s) are available.
         output_datasets = vars.get('out_data', [])
         rval = {'outputs': [], 'output_collections': [], 'jobs': [], 'implicit_collections': []}
-
+        rval['produces_entry_points'] = tool.produces_entry_points
         job_errors = vars.get('job_errors', [])
         if job_errors:
             # If we are here - some jobs were successfully executed but some failed.
@@ -557,7 +557,6 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
             output_dict['output_name'] = output_name
             rval['implicit_collections'].append(output_dict)
 
-        rval['view_result'] = vars.get('view_result', None)
         return rval
 
     def _patch_library_inputs(self, trans, inputs, target_history):
diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py
index ae81d132595..a42eb85f0ba 100644
--- a/lib/galaxy/webapps/galaxy/buildapp.py
+++ b/lib/galaxy/webapps/galaxy/buildapp.py
@@ -359,6 +359,7 @@ def populate_api_routes(webapp, app):
     webapp.mapper.resource('tool', 'tools', path_prefix='/api')
     webapp.mapper.resource('dynamic_tools', 'dynamic_tools', path_prefix='/api')
 
+    webapp.mapper.connect('/api/entry_points', action='index', controller="tool_entry_points")
     webapp.mapper.connect('/api/entry_points/{id:.+?}/access', action='access_entry_point', controller="tool_entry_points")
 
     webapp.mapper.connect('/api/dependency_resolvers/clean', action="clean", controller="tool_dependencies", conditions=dict(method=["POST"]))
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index ee7abaf7763..0902288ca7d 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -77,19 +77,6 @@ class RealTimeToolEntryPointListGrid(grids.Grid):
 class RealTime(BaseUIController):
     entry_point_grid = RealTimeToolEntryPointListGrid()
 
-    @web.expose
-    def index(self, trans, job_id=None, **kwd):
-        job = trans.sa_session.query(trans.app.model.Job).get(self.decode_id(job_id))
-        eps = job.realtimetool_entry_points
-        try:
-            if eps and len(eps) == 1:
-                redirect_target = RealTimeManager(self.app).access_entry_point_target(trans, eps[0].id)
-            else:
-                redirect_target = url_for(controller="realtime", action="list")
-            return trans.response.send_redirect(redirect_target)
-        except exceptions.MessageException as e:
-            return trans.show_error_message(str(e))
-
     @web.expose_api_anonymous
     def list(self, trans, **kwargs):
         """List all available realtimetools"""

From 03e9a6fe0b8b1306d681c0620b2f5190ac93397c Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Mon, 20 May 2019 14:11:45 -0400
Subject: [PATCH 065/114] Integration testing for RTT API hooks.

---
 lib/galaxy/tool_util/deps/docker_util.py      |   2 +
 test/base/driver_util.py                      |  30 ++++-
 .../tools/realtime/simple/Dockerfile          |   4 +
 .../functional/tools/realtime/simple/Makefile |   3 +
 test/functional/tools/realtimetool_simple.xml |  24 ++++
 .../tools/realtimetool_two_entry_points.xml   |  29 +++++
 test/functional/tools/samples_tool_conf.xml   |   3 +
 test/integration/test_containerized_jobs.py   |   9 ++
 test/integration/test_realtime_tools_api.py   | 104 ++++++++++++++++++
 9 files changed, 207 insertions(+), 1 deletion(-)
 create mode 100644 test/functional/tools/realtime/simple/Dockerfile
 create mode 100644 test/functional/tools/realtime/simple/Makefile
 create mode 100644 test/functional/tools/realtimetool_simple.xml
 create mode 100644 test/functional/tools/realtimetool_two_entry_points.xml
 create mode 100644 test/integration/test_realtime_tools_api.py

diff --git a/lib/galaxy/tool_util/deps/docker_util.py b/lib/galaxy/tool_util/deps/docker_util.py
index 9f53e4bda64..9647ca7ecc9 100644
--- a/lib/galaxy/tool_util/deps/docker_util.py
+++ b/lib/galaxy/tool_util/deps/docker_util.py
@@ -229,6 +229,8 @@ def parse_port_text(port_text):
     if port_text is not None:
         ports = {}
         for line in port_text.strip().split('\n'):
+            if " -> " not in line:
+                raise Exception("Cannot parse host and port from line [%s]" % line)
             tool, host = line.split(" -> ", 1)
             hostname, port = host.split(':')
             port = int(port)
diff --git a/test/base/driver_util.py b/test/base/driver_util.py
index c828c848389..8210cd580e2 100644
--- a/test/base/driver_util.py
+++ b/test/base/driver_util.py
@@ -29,6 +29,7 @@ from sqlalchemy_utils import (
     create_database,
     database_exists,
 )
+import yaml
 
 from galaxy.app import UniverseApplication as GalaxyUniverseApplication
 from galaxy.config import LOGGING_CONFIG_DEFAULT
@@ -59,6 +60,17 @@ MIGRATED_TOOL_PANEL_CONFIG = 'config/migrated_tools_conf.xml'
 INSTALLED_TOOL_PANEL_CONFIGS = [
     os.environ.get('GALAXY_TEST_SHED_TOOL_CONF', 'config/shed_tool_conf.xml')
 ]
+REALTIME_PROXY_TEMPLATE = string.Template(r"""
+uwsgi:
+  realtime_map: $tempdir/realtime_map.sqlite
+  python-raw: scripts/realtime/key_type_token_mapping.py
+  route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.$test_host:$test_port)$ goto:realtime
+  route-run: goto:endendend
+  route-label: realtime
+  route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.$test_host:$test_port)$ rpcvar:TARGET_HOST rtt_key_type_token_mapper_cached $2 $1 $3 $4 $0 5
+  route-if-not: empty:${TARGET_HOST} httpdumb:${TARGET_HOST}
+  route-label: endendend
+""")
 
 DEFAULT_LOCALES = "en"
 
@@ -713,11 +725,27 @@ def launch_uwsgi(kwargs, tempdir, prefix=DEFAULT_CONFIG_PREFIX, config_object=No
     config = {}
     config["galaxy"] = kwargs.copy()
 
+    enable_realtime_mapping = getattr(config_object, "enable_realtime_mapping", False)
+    if enable_realtime_mapping:
+        config["galaxy"]["realtime_prefix"] = "realtime"
+        config["galaxy"]["realtime_map"] = os.path.join(tempdir, "realtime_map.sqlite")
+
     yaml_config_path = os.path.join(tempdir, "galaxy.yml")
     with open(yaml_config_path, "w") as f:
-        import yaml
         yaml.dump(config, f)
 
+    if enable_realtime_mapping:
+        # Avoid YAML.dump configuration since uwsgi doesn't like real YAML :( -
+        # though maybe it would work?
+        with open(yaml_config_path, "r") as f:
+            old_contents = f.read()
+        with open(yaml_config_path, "w") as f:
+            test_port = str(port) if port else r"[0-9]+"
+            test_host = host or "localhost"
+            uwsgi_section = REALTIME_PROXY_TEMPLATE.safe_substitute(test_host=test_host, test_port=test_port, tempdir=tempdir)
+            f.write(uwsgi_section)
+            f.write(old_contents)
+
     def attempt_port_bind(port):
         uwsgi_command = [
             "uwsgi",
diff --git a/test/functional/tools/realtime/simple/Dockerfile b/test/functional/tools/realtime/simple/Dockerfile
new file mode 100644
index 00000000000..3593652582d
--- /dev/null
+++ b/test/functional/tools/realtime/simple/Dockerfile
@@ -0,0 +1,4 @@
+FROM python:2.7
+EXPOSE 7000
+EXPOSE 7001
+RUN echo 'moo cow' > /index.html
diff --git a/test/functional/tools/realtime/simple/Makefile b/test/functional/tools/realtime/simple/Makefile
new file mode 100644
index 00000000000..277a06e3d2a
--- /dev/null
+++ b/test/functional/tools/realtime/simple/Makefile
@@ -0,0 +1,3 @@
+
+all:
+	docker build . -t galaxyproject/test-http-example:0.1
diff --git a/test/functional/tools/realtimetool_simple.xml b/test/functional/tools/realtimetool_simple.xml
new file mode 100644
index 00000000000..1adff15b456
--- /dev/null
+++ b/test/functional/tools/realtimetool_simple.xml
@@ -0,0 +1,24 @@
+<tool id="realtimetool_simple" name="realtimetool_simple" tool_type="realtime" version="0.1">
+    <requirements>
+        <container type="docker">galaxyproject/test-http-example:0.1</container>
+    </requirements>
+    <realtime>
+        <entry_point name="Simple">
+            <port>7000</port>
+            <url>/</url>
+        </entry_point>
+    </realtime>
+    <command detect_errors="exit_code"><![CDATA[
+        cd /;
+        python -m SimpleHTTPServer 7000
+    ]]>
+    </command>
+    <inputs>
+    </inputs>
+    <outputs>
+    </outputs>
+    <tests>
+    </tests>
+    <help>
+    </help>
+</tool>
diff --git a/test/functional/tools/realtimetool_two_entry_points.xml b/test/functional/tools/realtimetool_two_entry_points.xml
new file mode 100644
index 00000000000..c9da3b3b1e2
--- /dev/null
+++ b/test/functional/tools/realtimetool_two_entry_points.xml
@@ -0,0 +1,29 @@
+<tool id="realtimetool_two_entry_points" name="realtimetool_two_entry_points" tool_type="realtime" version="0.1">
+    <requirements>
+        <container type="docker">galaxyproject/test-http-example:0.1</container>
+    </requirements>
+    <realtime>
+        <entry_point name="Server 1">
+            <port>7000</port>
+            <url>/</url>
+        </entry_point>
+        <entry_point name="Server 2">
+            <port>7001</port>
+            <url>/</url>
+        </entry_point>
+    </realtime>
+    <command detect_errors="exit_code"><![CDATA[
+        cd /;
+        python -m SimpleHTTPServer 7000 &
+        python -m SimpleHTTPServer 7001
+    ]]>
+    </command>
+    <inputs>
+    </inputs>
+    <outputs>
+    </outputs>
+    <tests>
+    </tests>
+    <help>
+    </help>
+</tool>
diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml
index 493582f31ff..f1e9bca3d20 100644
--- a/test/functional/tools/samples_tool_conf.xml
+++ b/test/functional/tools/samples_tool_conf.xml
@@ -164,6 +164,9 @@
   <tool file="multiple_versions_v01.xml" />
   <tool file="multiple_versions_v02.xml" />
 
+  <tool file="realtimetool_simple.xml" />
+  <tool file="realtimetool_two_entry_points.xml" />
+
   <!-- Tools interesting only for building up test workflows. -->
 
   <!-- Next three tools demonstrate concatenating multiple datasets
diff --git a/test/integration/test_containerized_jobs.py b/test/integration/test_containerized_jobs.py
index 24f6ede524d..7a83d2a595e 100644
--- a/test/integration/test_containerized_jobs.py
+++ b/test/integration/test_containerized_jobs.py
@@ -31,6 +31,15 @@ class MulledJobTestCases(object):
         assert "0.7.15-r1140" in output
 
 
+class ContainerizedIntegrationTestCase(integration_util.IntegrationTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        if not which(cls.container_type):
+            raise unittest.SkipTest("Executable '%s' not found on PATH" % cls.container_type)
+        super(ContainerizedIntegrationTestCase, cls).setUpClass()
+
+
 class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, RunsEnvironmentJobs, MulledJobTestCases):
 
     framework_tool_and_types = True
diff --git a/test/integration/test_realtime_tools_api.py b/test/integration/test_realtime_tools_api.py
new file mode 100644
index 00000000000..0e1a51966e6
--- /dev/null
+++ b/test/integration/test_realtime_tools_api.py
@@ -0,0 +1,104 @@
+"""Integration tests for realtime tools."""
+
+import os
+
+import requests
+
+from .test_containerized_jobs import ContainerizedIntegrationTestCase
+from base import api_asserts
+from base.populators import (
+    DatasetPopulator,
+    wait_on,
+)
+
+SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
+
+
+class RealtimeToolsIntegrationTestCase(ContainerizedIntegrationTestCase):
+
+    framework_tool_and_types = True
+    container_type = "docker"
+    require_uwsgi = True
+    enable_realtime_mapping = True
+
+    def setUp(self):
+        super(RealtimeToolsIntegrationTestCase, self).setUp()
+        self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
+        self.history_id = self.dataset_populator.new_history()
+
+    def test_simple_execution(self):
+        response_dict = self.dataset_populator.run_tool("realtimetool_simple", {}, self.history_id, assert_ok=True)
+        assert "jobs" in response_dict, response_dict
+        jobs = response_dict["jobs"]
+        assert isinstance(jobs, list)
+        assert len(jobs) == 1
+        job0 = jobs[0]
+        entry_points = self.wait_on_entry_points_active(job0["id"])
+        assert len(entry_points) == 1
+        entry_point0 = entry_points[0]
+        target = self.entry_point_target(entry_point0["id"])
+        content = self.wait_on_proxied_content(target)
+        assert content == "moo cow\n", content
+
+    def test_multi_server_realtime_tool(self):
+        response_dict = self.dataset_populator.run_tool("realtimetool_two_entry_points", {}, self.history_id, assert_ok=True)
+        assert "jobs" in response_dict, response_dict
+        jobs = response_dict["jobs"]
+        assert isinstance(jobs, list)
+        assert len(jobs) == 1
+        job0 = jobs[0]
+        entry_points = self.wait_on_entry_points_active(job0["id"])
+        assert len(entry_points) == 2
+        entry_point0 = entry_points[0]
+        entry_point1 = entry_points[1]
+        target0 = self.entry_point_target(entry_point0["id"])
+        target1 = self.entry_point_target(entry_point1["id"])
+        assert target0 != target1
+        content0 = self.wait_on_proxied_content(target0)
+        assert content0 == "moo cow\n", content0
+
+        content1 = self.wait_on_proxied_content(target1)
+        assert content1 == "moo cow\n", content1
+        assert False
+
+    def wait_on_proxied_content(self, target):
+        def get_hosted_content():
+            try:
+                scheme, rest = target.split("://", 1)
+                prefix, host_and_port = rest.split(".realtime.")
+                print(rest)
+                faked_host = rest
+                if "/" in rest:
+                    faked_host = rest.split("/", 1)[0]
+                response = requests.get("%s://%s" % (scheme, host_and_port), timeout=1, headers={"Host": faked_host})
+                return response.content
+            except Exception as e:
+                print(e)
+                return None
+
+        content = wait_on(get_hosted_content, "realtime hosted content at %s" % target)
+        return content
+
+    def entry_point_target(self, entry_point_id):
+        entry_point_access_response = self._get("entry_points/%s/access" % entry_point_id)
+        api_asserts.assert_status_code_is(entry_point_access_response, 200)
+        access_json = entry_point_access_response.json()
+        api_asserts.assert_has_key(access_json, "target")
+        return access_json["target"]
+
+    def wait_on_entry_points_active(self, job_id, expected_num=1):
+        def active_entry_points():
+            entry_points = self.entry_points_for_job(job_id)
+            if len(entry_points) != expected_num:
+                return None
+            elif any([not e["active"] for e in entry_points]):
+                return None
+            else:
+                return entry_points
+
+        return wait_on(active_entry_points, "entry points to become active")
+
+    def entry_points_for_job(self, job_id):
+        entry_points_response = self._get("entry_points?job_id=%s" % job_id)
+        api_asserts.assert_status_code_is(entry_points_response, 200)
+        return entry_points_response.json()

From ba41c5b35effb25dc1294b54614e7ba59e7a8a91 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 22 May 2019 08:18:52 -0400
Subject: [PATCH 066/114] Rebase into intergration testsing..

---
 test/functional/tools/realtime/simple/Makefile          | 2 +-
 test/functional/tools/realtimetool_simple.xml           | 2 +-
 test/functional/tools/realtimetool_two_entry_points.xml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/functional/tools/realtime/simple/Makefile b/test/functional/tools/realtime/simple/Makefile
index 277a06e3d2a..bab9fd896e4 100644
--- a/test/functional/tools/realtime/simple/Makefile
+++ b/test/functional/tools/realtime/simple/Makefile
@@ -1,3 +1,3 @@
 
 all:
-	docker build . -t galaxyproject/test-http-example:0.1
+	docker build . -t galaxy/test-http-example:0.1
diff --git a/test/functional/tools/realtimetool_simple.xml b/test/functional/tools/realtimetool_simple.xml
index 1adff15b456..5f20ec07c0f 100644
--- a/test/functional/tools/realtimetool_simple.xml
+++ b/test/functional/tools/realtimetool_simple.xml
@@ -1,6 +1,6 @@
 <tool id="realtimetool_simple" name="realtimetool_simple" tool_type="realtime" version="0.1">
     <requirements>
-        <container type="docker">galaxyproject/test-http-example:0.1</container>
+        <container type="docker">galaxy/test-http-example:0.1</container>
     </requirements>
     <realtime>
         <entry_point name="Simple">
diff --git a/test/functional/tools/realtimetool_two_entry_points.xml b/test/functional/tools/realtimetool_two_entry_points.xml
index c9da3b3b1e2..de5d439a4dc 100644
--- a/test/functional/tools/realtimetool_two_entry_points.xml
+++ b/test/functional/tools/realtimetool_two_entry_points.xml
@@ -1,6 +1,6 @@
 <tool id="realtimetool_two_entry_points" name="realtimetool_two_entry_points" tool_type="realtime" version="0.1">
     <requirements>
-        <container type="docker">galaxyproject/test-http-example:0.1</container>
+        <container type="docker">galaxy/test-http-example:0.1</container>
     </requirements>
     <realtime>
         <entry_point name="Server 1">

From 1126d68fedc32c1644915addce951a945496d776 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 22 May 2019 09:27:41 -0400
Subject: [PATCH 067/114] Rework client polling & entry point index API for
 potential reuse in the grid.

Fix links in the job monitor widget.
---
 .../ToolEntryPoints/ToolEntryPoints.vue       | 42 ++++++-------------
 client/galaxy/scripts/mvc/entrypoints/poll.js | 33 +++++++++++++++
 lib/galaxy/managers/realtime.py               | 17 ++++----
 .../webapps/galaxy/api/tool_entry_points.py   | 32 ++++++++++----
 4 files changed, 80 insertions(+), 44 deletions(-)
 create mode 100644 client/galaxy/scripts/mvc/entrypoints/poll.js

diff --git a/client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue b/client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue
index 2465785f2c7..a6cf8a124b9 100644
--- a/client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue
+++ b/client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue
@@ -5,7 +5,7 @@
         </p>
         <p v-else-if="entryPoints.length==1" >
             <span v-if="entryPoints[0].active" >
-                There is a RealTimeTool result view available, click here to display.
+                There is a RealTimeTool result view available, <a :href="entryPoints[0].target">click here to display</a>.
             </span>
             <span v-else>
                 There is a RealTimeTool result view available, waiting for view to become active...
@@ -17,7 +17,7 @@
                 <li v-for="entryPoint of entryPoints" v-bind:key="entryPoint.id" >
                     {{ entryPoint.name }}
                     <span v-if="entryPoint.active">
-                        (active)
+                        (<a :href="entryPoints[0].target">click here to display</a>)
                     </span>
                     <span v-else>
                         (waiting to become active...)
@@ -32,6 +32,7 @@
 
 <script>
 import { getAppRoot } from "onload/loadConfig";
+import { clearPolling, pollUntilActive } from "mvc/entrypoints/poll";
 import axios from "axios";
 export default {
     props: {
@@ -43,40 +44,23 @@ export default {
     data() {
         return {
             entryPoints: [],
-            interval: null,
         };
     },
     created: function() {
-        this.reloadEntryPoints();
+        this.pollEntryPoints();
     },
     beforeDestroy: function(){
-        clearInterval(this.interval);
+        clearPolling();
     },
     methods: {
-        reloadEntryPoints: function() {
-            clearInterval(this.interval);
-            const url = getAppRoot() + `api/entry_points?job_id=${this.jobId}`;
-            axios
-                .get(url)
-                .then(response => {
-                    const entryPoints = [];
-                    let allReady = true;
-                    response.data.forEach((entryPoint, i) => {
-                        entryPoints.push(entryPoint);
-                        if(! entryPoint.active) {
-                            allReady = false;
-                        }
-                    });
-                    this.entryPoints = entryPoints;
-                    if(! allReady || entryPoints.length == 0) {
-                        this.interval = setInterval(() => {
-                            this.reloadEntryPoints();
-                        }, 5000);
-                    }
-                })
-                .catch(e => {
-                    console.error(e);
-                });
+        pollEntryPoints: function() {
+            let onUpdate = (entryPoints) => {
+                this.entryPoints = entryPoints;
+            }
+            let onError = (e) => {
+                console.error(e);
+            }
+            pollUntilActive(onUpdate, onError, {"job_id": this.jobId})
         }
     }
 };
diff --git a/client/galaxy/scripts/mvc/entrypoints/poll.js b/client/galaxy/scripts/mvc/entrypoints/poll.js
new file mode 100644
index 00000000000..899b5e7ec9c
--- /dev/null
+++ b/client/galaxy/scripts/mvc/entrypoints/poll.js
@@ -0,0 +1,33 @@
+import axios from "axios";
+import { getAppRoot } from "onload/loadConfig";
+
+let interval;
+
+export const clearPolling = () => {
+    clearInterval(interval);
+}
+
+export const pollUntilActive = (onUpdate, onError, params) => {
+    clearPolling();
+    const url = getAppRoot() + `api/entry_points`;
+    console.log(params);
+    axios
+        .get(url, {params: params})
+        .then(response => {
+            const entryPoints = [];
+            let allReady = true;
+            response.data.forEach((entryPoint, i) => {
+                entryPoints.push(entryPoint);
+                if(! entryPoint.active) {
+                    allReady = false;
+                }
+            });
+            onUpdate(entryPoints)
+            if(! allReady || entryPoints.length == 0) {
+                interval = setInterval(() => {
+                    pollUntilActive(onUpdate, onError, params);
+                }, 3000);
+            }
+        })
+        .catch(onError);
+}
diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index 81bfdf44e31..53b94a4a2fc 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -279,17 +279,20 @@ class RealTimeManager(object):
             self.sa_session.flush()
         self.propagator.remove_entry_point(entry_point)
 
+    def target_if_active(self, trans, entry_point):
+        if entry_point.active and not entry_point.deleted:
+            request_host = trans.request.host
+            rval = '%s//%s.%s.%s.%s.%s/' % (trans.request.host_url.split('//', 1)[0], entry_point.__class__.__name__.lower(), trans.security.encode_id(entry_point.id),
+                    entry_point.token, self.app.config.realtime_prefix, request_host)
+            if entry_point.entry_url:
+                rval = '%s/%s' % (rval.rstrip('/'), entry_point.entry_url.lstrip('/'))
+            return rval
+
     def access_entry_point_target(self, trans, entry_point_id):
         entry_point = trans.sa_session.query(model.RealTimeToolEntryPoint).get(entry_point_id)
         if self.app.realtime_manager.can_access_entry_point(trans, entry_point):
             if entry_point.active:
-                request_host = trans.request.host
-                log.info("building access for host %s" % request_host)
-                rval = '%s//%s.%s.%s.%s.%s/' % (trans.request.host_url.split('//', 1)[0], entry_point.__class__.__name__.lower(), trans.security.encode_id(entry_point.id),
-                        entry_point.token, self.app.config.realtime_prefix, request_host)
-                if entry_point.entry_url:
-                    rval = '%s/%s' % (rval.rstrip('/'), entry_point.entry_url.lstrip('/'))
-                return rval
+                return self.target_if_active(entry_point)
             elif entry_point.deleted:
                 raise exceptions.MessageException('RealTimeTool has ended. You will have to start a new one.')
             else:
diff --git a/lib/galaxy/webapps/galaxy/api/tool_entry_points.py b/lib/galaxy/webapps/galaxy/api/tool_entry_points.py
index 4ba72dde997..7b9dd0698fc 100644
--- a/lib/galaxy/webapps/galaxy/api/tool_entry_points.py
+++ b/lib/galaxy/webapps/galaxy/api/tool_entry_points.py
@@ -3,9 +3,8 @@ related to running and queued jobs.
 """
 import logging
 
-from galaxy import exceptions
+from galaxy import exceptions, util
 from galaxy.managers.realtime import RealTimeManager
-from galaxy.util.dictifiable import dict_for
 from galaxy.web import expose_api_anonymous_and_sessionless
 from galaxy.web.base.controller import BaseAPIController
 
@@ -19,7 +18,7 @@ class ToolEntryPointsAPIController(BaseAPIController):
         self.realtime_manager = RealTimeManager(app)
 
     @expose_api_anonymous_and_sessionless
-    def index(self, trans, job_id=None, **kwd):
+    def index(self, trans, running=False, job_id=None, **kwd):
         """
         * GET /api/entry_points
             Returns tool entry point information. Currently passing a job_id
@@ -29,17 +28,34 @@ class ToolEntryPointsAPIController(BaseAPIController):
         :type   job_id: string
         :param  job_id: Encoded job id
 
+        :type   running: boolean
+        :param  running: filter to only include running job entry points.
+
         :rtype:     list
         :returns:   list of entry point dictionaries.
         """
-        job = trans.sa_session.query(trans.app.model.Job).get(self.decode_id(job_id))
-        if not self.realtime_manager.can_access_job(trans, job):
-            raise exceptions.ItemAccessibilityException()
-        entry_points = job.realtimetool_entry_points
+        running = util.asbool(running)
+        if job_id is None and not running:
+            raise exceptions.RequestParameterInvalidException("Currently this API must passed a job id or running=true")
+
+        if job_id is not None and running:
+            raise exceptions.RequestParameterInvalidException("Currently this API must passed only a job id or running=true")
+
+        if job_id is not None:
+            job = trans.sa_session.query(trans.app.model.Job).get(self.decode_id(job_id))
+            if not self.realtime_manager.can_access_job(trans, job):
+                raise exceptions.ItemAccessibilityException()
+            entry_points = job.realtimetool_entry_points
+        if running:
+            entry_points = self.realtime_manager.get_nonterminal_for_user_by_trans(trans)
 
         rval = []
         for entry_point in entry_points:
-            rval.append(self.encode_all_ids(trans, entry_point.to_dict(), True))
+            as_dict = self.encode_all_ids(trans, entry_point.to_dict(), True)
+            target = self.realtime_manager.target_if_active(trans, entry_point)
+            if target:
+                as_dict["target"] = target
+            rval.append(as_dict)
         return rval
 
     @expose_api_anonymous_and_sessionless

From 75c57854b5642c6fa2065141f08d933a2161d6ba Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 22 May 2019 12:56:51 -0400
Subject: [PATCH 068/114] Make realtime tool grid actual links, poll on active.

---
 client/galaxy/scripts/mvc/entrypoints/view.js | 43 +++++++++++--------
 .../webapps/galaxy/controllers/realtime.py    | 16 +------
 2 files changed, 26 insertions(+), 33 deletions(-)

diff --git a/client/galaxy/scripts/mvc/entrypoints/view.js b/client/galaxy/scripts/mvc/entrypoints/view.js
index 63789baba71..3d8a0ce2b3e 100644
--- a/client/galaxy/scripts/mvc/entrypoints/view.js
+++ b/client/galaxy/scripts/mvc/entrypoints/view.js
@@ -1,27 +1,34 @@
 import $ from "jquery";
-import axios from "axios";
 import GridView from "mvc/grid/grid-view";
-import { getAppRoot } from "onload";
-import { Toast } from "ui/toast";
+import { clearPolling, pollUntilActive } from "mvc/entrypoints/poll";
+
 
 export default GridView.extend({
     init_grid_elements: function() {
         GridView.prototype.init_grid_elements.call(this);
-        $(".entry-point-link").click(event => {
-            const $link = $(event.target);
-            const entry_point_id = $link.attr("entry_point_id");
-            const url = `${getAppRoot()}api/entry_points/${entry_point_id}/access`;
-            const handlerError = err => {
-                let message = "Problem finding entry point target, please contact an admin.";
-                if (err.response) {
-                    message = err.response.data.err_msg;
+
+        const activated = {};
+
+        let onUpdate = (entryPoints) => {
+            entryPoints.forEach((entryPoint) => {
+                const entryPointId = entryPoint.id;
+                if (entryPoint.active && ! activated[entryPointId]) {
+                    const $link = $(`.entry-point-link[entry_point_id='${entryPointId}']`)
+                    if ($link.length > 0) {
+                        $link.attr("href", entryPoint["target"]);
+                        activated[entryPointId] = true;
+                    }
                 }
-                Toast.error(message);
-            };
-            axios
-                .get(url)
-                .then(response => (window.location = response.data.target))
-                .catch(handlerError);
-        });
+            });
+        }
+        let onError = (e) => {
+            console.error(e);
+        }
+        pollUntilActive(onUpdate, onError, {"running": true});
+    },
+    remove: function() {
+        // Your processing code here
+        clearPolling();
+        GridView.prototype.remove.apply(this, arguments);
     }
 });
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index 0902288ca7d..062a1da25a5 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -28,23 +28,10 @@ class JobStatusColumn(grids.StateColumn):
 
 class EntryPointLinkColumn(grids.GridColumn):
     def get_value(self, trans, grid, item):
-        return '<a class="entry-point-link" entry_point_id="%s">%s</div>' % (trans.security.encode_id(item.id), item.name)
+        return '<a class="entry-point-link" entry_point_id="%s">%s</a>' % (trans.security.encode_id(item.id), item.name)
 
 
 class RealTimeToolEntryPointListGrid(grids.Grid):
-    def get_url_args(item):
-        """
-        Returns dictionary used to create item link.
-        """
-        url_kwargs = dict(controller="realtime", action="access_entry_point", id=item.id)
-        return url_kwargs
-
-    def get_url_args_job(item):
-        """
-        Returns dictionary used to create item link.
-        """
-        url_kwargs = dict(controller="api/jobs", action='show', id=item.job.id)
-        return url_kwargs
 
     use_panels = True
     title = "Available RealTimeTools"
@@ -53,7 +40,6 @@ class RealTimeToolEntryPointListGrid(grids.Grid):
     default_sort_key = "-update_time"
     columns = [
         EntryPointLinkColumn("Name", filterable="advanced"),
-        grids.GridColumn("Active", key="active"),
         JobStatusColumn("Job Info", key="job_state", model_class=model.Job),
         grids.GridColumn("Created", key="created_time", format=time_ago),
         grids.GridColumn("Last Updated", key="modified_time", format=time_ago),

From 0dfe1343681db04223c4f5696649a2a5dca0c9a4 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 22 May 2019 16:09:05 -0400
Subject: [PATCH 069/114] Realtime Tools: Monitor for entrypoint ports in job
 script.

A few more pieces for local execution but more general, only requires a shared file system with Galaxy instead of direct access to docker host the job is running inside. Allows us to skip populating 'port' command in job destination. Eliminates a lot of condor hackiness - that implementation shares a lot more code with the local runner now.
---
 lib/galaxy/jobs/__init__.py                   | 45 +++++++++++++++++++
 lib/galaxy/jobs/command_factory.py            |  4 ++
 lib/galaxy/jobs/runners/condor.py             | 33 +++-----------
 lib/galaxy/jobs/runners/local.py              | 44 ++++--------------
 lib/galaxy/managers/realtime.py               |  6 +--
 lib/galaxy/model/__init__.py                  |  8 ++++
 .../tool_util/deps/container_classes.py       | 10 ++++-
 7 files changed, 81 insertions(+), 69 deletions(-)

diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py
index 1825d217916..5c4964c96e4 100644
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -4,6 +4,7 @@ Support for running a tool in Galaxy via an internal job management system
 import copy
 import datetime
 import errno
+import json
 import logging
 import os
 import pwd
@@ -1973,6 +1974,50 @@ class JobWrapper(HasResourceParameters):
                     command = "%s; %s" % (dependency_shell_commands, command)
         return command
 
+    def check_for_entry_points(self, check_already_configured=True):
+        if not self.tool.produces_entry_points:
+            return True
+
+        job = self.get_job()
+        if check_already_configured and job.all_entry_points_configured:
+            return True
+
+        working_directory = self.working_directory
+        container_runtime_path = os.path.join(working_directory, "container_runtime.json")
+        if os.path.exists(container_runtime_path):
+            with open(container_runtime_path, "r") as f:
+                container_runtime = json.load(f)
+            log.debug("found container runtime %s" % container_runtime)
+            self.app.realtime_manager.configure_entry_points(job, container_runtime)
+            return True
+
+    def container_montior_command(self, container, **kwds):
+        if not container or not self.tool.produces_entry_points:
+            return None
+
+        from os.path import abspath
+        from os import getcwd
+        exec_dir = kwds.get('exec_dir', abspath(getcwd()))
+        work_dir = self.working_directory
+        container_config = os.path.join(work_dir, "container_config.json")
+
+        # TODO: implement callback via URL callback for configuring ports instead
+        #       of fs polling.
+        # if self.app.config.galaxy_infrastructure_url_set:
+        #    # TODO: settable from job destination...
+        #    infrastructure_url = self.app.config.galaxy_infrastructure_url
+        # else:
+        #    infrastructure_url = "host.docker.internal"
+
+        with open(container_config, "w") as f:
+            json.dump({
+                "container_name": container.container_name,
+                "container_type": container.container_type,
+                "connection_configuration": container.connection_configuration,
+            }, f)
+
+        return "(python '%s'/lib/galaxy_ext/container_monitor/monitor.py &) " % exec_dir
+
     @property
     def user(self):
         job = self.get_job()
diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py
index ad99b0e1e56..ef1701ba652 100644
--- a/lib/galaxy/jobs/command_factory.py
+++ b/lib/galaxy/jobs/command_factory.py
@@ -103,6 +103,10 @@ def build_command(
         # xref https://github.com/galaxyproject/galaxy/issues/3289
         commands_builder.prepend_command("rm -rf working; mkdir -p working; cd working")
 
+    container_montior_command = job_wrapper.container_montior_command(container)
+    if container_montior_command:
+        commands_builder.prepend_command(container_montior_command)
+
     if include_work_dir_outputs:
         __handle_work_dir_outputs(commands_builder, job_wrapper, runner, remote_command_params)
 
diff --git a/lib/galaxy/jobs/runners/condor.py b/lib/galaxy/jobs/runners/condor.py
index 019ce72083f..df68f34b11c 100644
--- a/lib/galaxy/jobs/runners/condor.py
+++ b/lib/galaxy/jobs/runners/condor.py
@@ -167,9 +167,6 @@ class CondorJobRunner(AsynchronousJobRunner):
         # Add to our 'queue' of jobs to monitor
         self.monitor_queue.put(cjs)
 
-        log.info('########### CondorExternalJobID befor handle_container %s' % external_job_id)
-        self.__handle_container(job_wrapper, external_job_id)
-
     def check_watched_items(self):
         """
         Called by the monitor thread to look at each watched job and deal
@@ -196,6 +193,11 @@ class CondorJobRunner(AsynchronousJobRunner):
                 cjs.fail_message = "Cluster could not complete job"
                 self.work_queue.put((self.fail_job, cjs))
                 continue
+
+            if job_running:
+                # If running, check for entry points...
+                cjs.job_wrapper.job_wrapper.check_for_entry_points()
+
             if job_running and not cjs.running:
                 log.debug("(%s/%s) job is now running" % (galaxy_id_tag, job_id))
                 cjs.job_wrapper.change_state(model.Job.states.RUNNING)
@@ -316,28 +318,3 @@ class CondorJobRunner(AsynchronousJobRunner):
         #                            preexec_fn=os.setpgrp)
         log.debug('_run_command(%s) exit code (%s) and failure: %s', command, exit_code, stderr)
         return (exit_code, ret)
-
-    def __handle_container(self, job_wrapper, external_job_id):
-        job = job_wrapper.get_job()
-        if job:
-            cont = job.container
-            if cont:
-                if cont.container_type == 'docker':
-                    # Handle getting RealTimeTool ports, if applicable
-                    eps = job.realtimetool_entry_points
-                    if eps:
-                        max_command_attempts = 10 + 1
-                        ports_raw = None
-                        for i in range(1, max_command_attempts):
-                            exit_code, ports_raw = self._run_command(cont.container_info['commands']['port'], external_job_id)
-                            if exit_code == 0:
-                                break
-                            if i != max_command_attempts:
-                                t = 2 * i
-                                log.debug("Container not found during port check, sleeping for %s seconds.", t)
-                                time.sleep(t)
-                        log.warn('\n\n RAW ports: (%s)' % ports_raw)
-                        if ports_raw is not None:
-                            self.app.realtime_manager.configure_entry_points_raw_docker_ports(job, ports_raw)
-                        else:
-                            log.error('Unable to run port command (%s): %s', cont.container_info['commands']['port'], exit_code)
diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py
index 45674d2ce51..5bacf8693ae 100644
--- a/lib/galaxy/jobs/runners/local.py
+++ b/lib/galaxy/jobs/runners/local.py
@@ -107,7 +107,7 @@ class LocalJobRunner(BaseJobRunner):
                 job_wrapper.set_job_destination(job_wrapper.job_destination, proc.pid)
                 job_wrapper.change_state(model.Job.states.RUNNING)
 
-                self.__handle_container(job_wrapper, proc)
+                self._handle_container(job_wrapper, proc)
 
                 terminated = self.__poll_if_needed(proc, job_wrapper, job_id)
                 if terminated:
@@ -225,8 +225,6 @@ class LocalJobRunner(BaseJobRunner):
             return False
 
     def _terminate(self, proc):
-        if proc.container_commands:
-            self._run_command(proc.container_commands['kill'])
         os.killpg(proc.pid, 15)
         sleep(1)
         if proc.poll() is None:
@@ -255,39 +253,15 @@ class LocalJobRunner(BaseJobRunner):
         log.debug('_run_command(%s) exit code %s', command, exit_code)
         return exit_code
 
-    def __handle_container(self, job_wrapper, proc):
-        proc.container_commands = None
-        job = job_wrapper.get_job()
-        if job:
-            cont = job.container
-            if cont:
-                if cont.container_type == 'docker':
-                    # Handle getting RealTimeTool ports, if applicable
-                    eps = job.realtimetool_entry_points
-                    if eps:
-                        max_command_attempts = 10 + 1
-                        ports_raw = None
-                        for i in range(1, max_command_attempts):
-                            with tempfile.TemporaryFile() as stdout_file:
-                                exit_code = subprocess.call(cont.container_info['commands']['port'],
-                                                        shell=True,
-                                                        stdout=stdout_file,
-                                                        env=self._environ,
-                                                        preexec_fn=os.setpgrp)
-                                if exit_code == 0:
-                                    stdout_file.seek(0)
-                                    ports_raw = stdout_file.read()
-                                    break
-                            if i != max_command_attempts:
-                                t = 2 * i
-                                log.debug("Container not found during port check, sleeping for %s seconds.", t)
-                                sleep(t)
+    def _handle_container(self, job_wrapper, proc):
+        if not job_wrapper.tool.produces_entry_points:
+            return
 
-                        if ports_raw is not None:
-                            self.app.realtime_manager.configure_entry_points_raw_docker_ports(job, ports_raw)
-                        else:
-                            log.error('Unable to run port command (%s): %s', cont.container_info['commands']['port'], exit_code)
-                    proc.container_commands = cont.container_info['commands']
+        while proc.poll() is None:
+            if job_wrapper.check_for_entry_points(check_already_configured=False):
+                return
+
+            sleep(0.5)
 
     def __poll_if_needed(self, proc, job_wrapper, job_id):
         # Only poll if needed (i.e. job limits are set)
diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index 53b94a4a2fc..67d0f0d3370 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -172,7 +172,7 @@ class RealTimeManager(object):
         configured = []
         not_configured = []
         for ep in job.realtimetool_entry_points:
-            port_dict = ports_dict.get(ep.tool_port, None)
+            port_dict = ports_dict.get(str(ep.tool_port), None)
             if port_dict is None:
                 log.error("Did not find port to assign to RealTimeToolEntryPoint by tool port: %s.", ep.tool_port)
                 not_configured.append(ep)
@@ -188,10 +188,6 @@ class RealTimeManager(object):
             self.sa_session.flush()
         return dict(not_configured=not_configured, configured=configured)
 
-    def configure_entry_points_raw_docker_ports(self, job, port_text):
-        ports_dict = docker_parse_port_text(port_text)
-        return self.configure_entry_points(job, ports_dict)
-
     def save_entry_point(self, entry_point):
         """
         Writeout a key, key_type, token, value store that is used validating access
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index b4399cfcb6c..f37dda4fd8a 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -924,6 +924,14 @@ class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById):
     def add_post_job_action(self, pja):
         self.post_job_actions.append(PostJobActionAssociation(pja, self))
 
+    @property
+    def all_entry_points_configured(self):
+        # consider an actual DB attribute for this.
+        all_configured = True
+        for ep in self.realtimetool_entry_points:
+            all_configured = ep.configured and all_configured
+        return all_configured
+
     def set_state(self, state):
         """
         Save state history
diff --git a/lib/galaxy/tool_util/deps/container_classes.py b/lib/galaxy/tool_util/deps/container_classes.py
index 899175e6492..73f974afb59 100644
--- a/lib/galaxy/tool_util/deps/container_classes.py
+++ b/lib/galaxy/tool_util/deps/container_classes.py
@@ -223,6 +223,10 @@ class DockerContainer(Container, HasDockerLikeVolumes):
         )
         return docker_host_props
 
+    @property
+    def connection_configuration(self):
+        return self.docker_host_props
+
     def build_pull_command(self):
         return docker_util.build_pull_command(self.container_id, **self.docker_host_props)
 
@@ -278,7 +282,7 @@ class DockerContainer(Container, HasDockerLikeVolumes):
             **docker_host_props
         )
         self.container_info['commands'] = {}
-        for cmd in ['inspect', 'port', 'kill', 'stop']:
+        for cmd in ['inspect', 'kill', 'stop']:
             self.container_info['commands'][cmd] = docker_util.build_docker_simple_command(cmd, container_name=self.container_name, **docker_host_props)
         return "%s\n%s" % (cache_command, run_command)
 
@@ -323,6 +327,10 @@ class SingularityContainer(Container, HasDockerLikeVolumes):
             sudo_cmd=self.prop("sudo_cmd", singularity_util.DEFAULT_SUDO_COMMAND),
         )
 
+    @property
+    def connection_configuration(self):
+        return self.get_singularity_target_kwds()
+
     def build_mulled_singularity_pull_command(self, cache_directory, namespace="biocontainers"):
         return singularity_util.pull_mulled_singularity_command(
             docker_image_identifier=self.container_id,

From 70989ef32694dfbbd4bc3e63b0ac72624f004fb8 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 22 May 2019 18:58:08 -0400
Subject: [PATCH 070/114] Proper container killing when running jobs.

Rip out container_info container hacks for interactive tools, container killing should now just work with local job runner (or any job runners).
---
 lib/galaxy/jobs/runners/__init__.py           |  7 +---
 lib/galaxy/jobs/runners/local.py              | 32 -------------------
 .../tool_util/deps/container_classes.py       | 11 ++++---
 3 files changed, 8 insertions(+), 42 deletions(-)

diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py
index d52d9d5bbe1..79767db5129 100644
--- a/lib/galaxy/jobs/runners/__init__.py
+++ b/lib/galaxy/jobs/runners/__init__.py
@@ -260,7 +260,7 @@ class BaseJobRunner(object):
         container = self._find_container(job_wrapper)
         if not container and job_wrapper.requires_containerization:
             raise Exception("Failed to find a container when required, contact Galaxy admin.")
-        built = build_command(
+        return build_command(
             self,
             job_wrapper,
             include_metadata=include_metadata,
@@ -270,11 +270,6 @@ class BaseJobRunner(object):
             stdout_file=stdout_file,
             stderr_file=stderr_file,
         )
-        stored_container = job_wrapper.get_job().container
-        if container and stored_container:
-            if container.container_info:
-                stored_container.container_info = container.container_info
-        return built
 
     def get_work_dir_outputs(self, job_wrapper, job_working_directory=None, tool_working_directory=None):
         """
diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py
index 5bacf8693ae..8a46e81fc46 100644
--- a/lib/galaxy/jobs/runners/local.py
+++ b/lib/galaxy/jobs/runners/local.py
@@ -147,15 +147,6 @@ class LocalJobRunner(BaseJobRunner):
         # if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished
         job = job_wrapper.get_job()
         job_ext_output_metadata = job.get_external_output_metadata()
-        if job.container:
-            try:
-                self._stop_container(job_wrapper)
-            except Exception as e:
-                log.warning("stop_job(): %s: trying to stop container failed. (%s)" % (job.id, e))
-                try:
-                    self._kill_container(job_wrapper)
-                except Exception as e:
-                    log.warning("stop_job(): %s: trying to kill container failed. (%s)" % (job.id, e))
         try:
             pid = job_ext_output_metadata[0].job_runner_external_pid  # every JobExternalOutputMetadata has a pid set, we just need to take from one of them
             assert pid not in [None, '']
@@ -181,7 +172,6 @@ class LocalJobRunner(BaseJobRunner):
                 return
         else:
             log.warning("stop_job(): %s: PID %d refuses to die after signaling TERM/KILL" % (job.id, pid))
-        self._kill_container(job_wrapper)
 
     def recover(self, job, job_wrapper):
         # local jobs can't be recovered
@@ -231,28 +221,6 @@ class LocalJobRunner(BaseJobRunner):
             os.killpg(proc.pid, 9)
         return proc.wait()  # reap
 
-    def _stop_container(self, job_wrapper):
-        return self._run_container_command(job_wrapper, 'stop')
-
-    def _kill_container(self, job_wrapper):
-        return self._run_container_command(job_wrapper, 'kill')
-
-    def _run_container_command(self, job_wrapper, command):
-        job = job_wrapper.get_job()
-        if job:
-            cont = job.container
-            if cont:
-                if cont.container_type == 'docker':
-                    return self._run_command(cont.container_info['commands'][command])
-
-    def _run_command(self, command):
-        exit_code = subprocess.call(command,
-                                    shell=True,
-                                    env=self._environ,
-                                    preexec_fn=os.setpgrp)
-        log.debug('_run_command(%s) exit code %s', command, exit_code)
-        return exit_code
-
     def _handle_container(self, job_wrapper, proc):
         if not job_wrapper.tool.produces_entry_points:
             return
diff --git a/lib/galaxy/tool_util/deps/container_classes.py b/lib/galaxy/tool_util/deps/container_classes.py
index 73f974afb59..ae0684b66b9 100644
--- a/lib/galaxy/tool_util/deps/container_classes.py
+++ b/lib/galaxy/tool_util/deps/container_classes.py
@@ -281,10 +281,13 @@ class DockerContainer(Container, HasDockerLikeVolumes):
             container_name=self.container_name,
             **docker_host_props
         )
-        self.container_info['commands'] = {}
-        for cmd in ['inspect', 'kill', 'stop']:
-            self.container_info['commands'][cmd] = docker_util.build_docker_simple_command(cmd, container_name=self.container_name, **docker_host_props)
-        return "%s\n%s" % (cache_command, run_command)
+        kill_command = docker_util.build_docker_simple_command("kill", container_name=self.container_name, **docker_host_props)
+        return """
+_on_exit() {
+  %s
+}
+trap _on_exit 0
+%s\n%s""" % (kill_command, cache_command, run_command)
 
     def __cache_from_file_command(self, cached_image_file, docker_host_props):
         images_cmd = docker_util.build_docker_images_command(truncate=False, **docker_host_props)

From e7d4265aafe0512f0c1eb160a8684ee2f607b20b Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 28 May 2019 09:14:32 -0400
Subject: [PATCH 071/114] Add missing container_monitor package to previous
 commits.

---
 lib/galaxy_ext/container_monitor/__ini__.py |  0
 lib/galaxy_ext/container_monitor/monitor.py | 48 +++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 lib/galaxy_ext/container_monitor/__ini__.py
 create mode 100644 lib/galaxy_ext/container_monitor/monitor.py

diff --git a/lib/galaxy_ext/container_monitor/__ini__.py b/lib/galaxy_ext/container_monitor/__ini__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/lib/galaxy_ext/container_monitor/monitor.py b/lib/galaxy_ext/container_monitor/monitor.py
new file mode 100644
index 00000000000..b12d23d0fc2
--- /dev/null
+++ b/lib/galaxy_ext/container_monitor/monitor.py
@@ -0,0 +1,48 @@
+import json
+import os
+import subprocess
+import sys
+import tempfile
+
+# insert *this* galaxy before all others on sys.path
+sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)))
+
+from galaxy.tool_util.deps import docker_util
+
+
+def main():
+    with open("container_config.json", "r") as f:
+        container_config = json.load(f)
+
+    container_type = container_config["container_type"]
+    container_name = container_config["container_name"]
+    connection_configuration = container_config["connection_configuration"]
+    if container_type != "docker":
+        raise Exception("Monitoring container type [%s], not yet implemented." % container_type)
+
+    ports_raw = None
+    try:
+        while True:
+            ports_command = docker_util.build_docker_simple_command("port", container_name=container_name, **connection_configuration)
+            with tempfile.TemporaryFile() as stdout_file:
+                exit_code = subprocess.call(ports_command,
+                                            shell=True,
+                                            stdout=stdout_file,
+                                            preexec_fn=os.setpgrp)
+                if exit_code == 0:
+                    stdout_file.seek(0)
+                    ports_raw = stdout_file.read()
+                    break
+
+        if ports_raw is not None:
+            with open("container_runtime.json", "w") as f:
+                json.dump(docker_util.parse_port_text(ports_raw), f)
+        else:
+            raise Exception("Failed to recover ports...")
+    except Exception as e:
+        with open("exception.txt", "w") as f:
+            f.write(str(e))
+
+
+if __name__ == "__main__":
+    main()

From 57ac6bd04dcc0f54fe768f8cc0382feee4a93f15 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Thu, 30 May 2019 16:40:54 -0400
Subject: [PATCH 072/114] Lowercase import Vue from "vue";

---
 client/galaxy/scripts/mvc/tool/tool-form.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js
index ab479f03a57..39fa0d63b76 100644
--- a/client/galaxy/scripts/mvc/tool/tool-form.js
+++ b/client/galaxy/scripts/mvc/tool/tool-form.js
@@ -11,7 +11,7 @@ import Ui from "mvc/ui/ui-misc";
 import Modal from "mvc/ui/ui-modal";
 import ToolFormBase from "mvc/tool/tool-form-base";
 import Webhooks from "mvc/webhooks";
-import Vue from "Vue";
+import Vue from "vue";
 import ToolEntryPoints from "components/ToolEntryPoints/ToolEntryPoints";
 
 var View = Backbone.View.extend({

From 1ce76c246e07e9e802d3c4acf3b14f27ad19ec5a Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Thu, 30 May 2019 18:02:22 -0400
Subject: [PATCH 073/114] flake8

---
 lib/galaxy/jobs/runners/condor.py                 | 1 -
 lib/galaxy/managers/realtime.py                   | 1 -
 lib/galaxy/webapps/galaxy/controllers/realtime.py | 3 ---
 test/base/driver_util.py                          | 2 +-
 test/integration/test_realtime_tools_api.py       | 4 +++-
 5 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/lib/galaxy/jobs/runners/condor.py b/lib/galaxy/jobs/runners/condor.py
index df68f34b11c..5a294c3a025 100644
--- a/lib/galaxy/jobs/runners/condor.py
+++ b/lib/galaxy/jobs/runners/condor.py
@@ -13,7 +13,6 @@ it at this time.
 import logging
 import os
 import subprocess
-import time
 
 from galaxy import model
 from galaxy.jobs.runners import (
diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index 67d0f0d3370..9f8ac577fd4 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -10,7 +10,6 @@ from galaxy import (
     model,
     exceptions
 )
-from galaxy.tool_util.deps.docker_util import parse_port_text as docker_parse_port_text
 from galaxy.util.filelock import FileLock
 
 
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index 062a1da25a5..e9af1f50737 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -4,12 +4,9 @@ Provides web interaction with RealTimeTools
 import logging
 
 from galaxy import (
-    exceptions,
     model,
     web
 )
-from galaxy.managers.realtime import RealTimeManager
-from galaxy.web import url_for
 from galaxy.web.base.controller import (
     BaseUIController,
 )
diff --git a/test/base/driver_util.py b/test/base/driver_util.py
index 8210cd580e2..9b2b2e8b5cb 100644
--- a/test/base/driver_util.py
+++ b/test/base/driver_util.py
@@ -14,6 +14,7 @@ import sys
 import tempfile
 import threading
 import time
+import yaml
 
 import nose.config
 import nose.core
@@ -29,7 +30,6 @@ from sqlalchemy_utils import (
     create_database,
     database_exists,
 )
-import yaml
 
 from galaxy.app import UniverseApplication as GalaxyUniverseApplication
 from galaxy.config import LOGGING_CONFIG_DEFAULT
diff --git a/test/integration/test_realtime_tools_api.py b/test/integration/test_realtime_tools_api.py
index 0e1a51966e6..6c6fc2d585d 100644
--- a/test/integration/test_realtime_tools_api.py
+++ b/test/integration/test_realtime_tools_api.py
@@ -4,8 +4,10 @@ import os
 
 import requests
 
-from .test_containerized_jobs import ContainerizedIntegrationTestCase
 from base import api_asserts
+
+from .test_containerized_jobs import ContainerizedIntegrationTestCase
+
 from base.populators import (
     DatasetPopulator,
     wait_on,

From e194ba44f6776957e65658a34a3946956b8ec3e4 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Thu, 30 May 2019 18:09:20 -0400
Subject: [PATCH 074/114] jslint

---
 .../components/ToolEntryPoints/ToolEntryPoints.vue        | 8 +++-----
 client/galaxy/scripts/mvc/entrypoints/view.js             | 4 ++--
 client/galaxy/scripts/mvc/tool/tool-form.js               | 8 ++++----
 3 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue b/client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue
index a6cf8a124b9..8e74d066724 100644
--- a/client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue
+++ b/client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue
@@ -31,9 +31,7 @@
 </template>
 
 <script>
-import { getAppRoot } from "onload/loadConfig";
 import { clearPolling, pollUntilActive } from "mvc/entrypoints/poll";
-import axios from "axios";
 export default {
     props: {
         jobId: {
@@ -54,14 +52,14 @@ export default {
     },
     methods: {
         pollEntryPoints: function() {
-            let onUpdate = (entryPoints) => {
+            const onUpdate = (entryPoints) => {
                 this.entryPoints = entryPoints;
             }
-            let onError = (e) => {
+            const onError = (e) => {
                 console.error(e);
             }
             pollUntilActive(onUpdate, onError, {"job_id": this.jobId})
         }
     }
 };
-</script>
\ No newline at end of file
+</script>
diff --git a/client/galaxy/scripts/mvc/entrypoints/view.js b/client/galaxy/scripts/mvc/entrypoints/view.js
index 3d8a0ce2b3e..c74af3435a1 100644
--- a/client/galaxy/scripts/mvc/entrypoints/view.js
+++ b/client/galaxy/scripts/mvc/entrypoints/view.js
@@ -9,7 +9,7 @@ export default GridView.extend({
 
         const activated = {};
 
-        let onUpdate = (entryPoints) => {
+        const onUpdate = (entryPoints) => {
             entryPoints.forEach((entryPoint) => {
                 const entryPointId = entryPoint.id;
                 if (entryPoint.active && ! activated[entryPointId]) {
@@ -21,7 +21,7 @@ export default GridView.extend({
                 }
             });
         }
-        let onError = (e) => {
+        const onError = (e) => {
             console.error(e);
         }
         pollUntilActive(onUpdate, onError, {"running": true});
diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js
index 39fa0d63b76..9ebde5ebcc5 100644
--- a/client/galaxy/scripts/mvc/tool/tool-form.js
+++ b/client/galaxy/scripts/mvc/tool/tool-form.js
@@ -236,11 +236,11 @@ var View = Backbone.View.extend({
                 callback && callback();
                 self.$el.children().hide();
                 if (response.produces_entry_points) {
-                    for (let job of response.jobs) {
-                        let toolEntryPointsInstance = Vue.extend(ToolEntryPoints);
-                        let vm = document.createElement("div");
+                    for (const job of response.jobs) {
+                        const toolEntryPointsInstance = Vue.extend(ToolEntryPoints);
+                        const vm = document.createElement("div");
                         self.$el.append(vm);
-                        let instance = new toolEntryPointsInstance({
+                        const instance = new toolEntryPointsInstance({
                             propsData: {
                                 jobId: job.id
                             }

From afc048dd64041ac2284bf4be16831b3236e331a5 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Thu, 30 May 2019 18:47:53 -0400
Subject: [PATCH 075/114] flake

---
 lib/galaxy/managers/realtime.py             | 4 ++--
 test/base/driver_util.py                    | 3 ++-
 test/integration/test_realtime_tools_api.py | 6 ++----
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index 9f8ac577fd4..d050a3593f6 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -7,8 +7,8 @@ from sqlalchemy import or_
 
 
 from galaxy import (
-    model,
-    exceptions
+    exceptions,
+    model
 )
 from galaxy.util.filelock import FileLock
 
diff --git a/test/base/driver_util.py b/test/base/driver_util.py
index 9b2b2e8b5cb..e98ff07fff1 100644
--- a/test/base/driver_util.py
+++ b/test/base/driver_util.py
@@ -14,12 +14,13 @@ import sys
 import tempfile
 import threading
 import time
-import yaml
 
 import nose.config
 import nose.core
 import nose.loader
 import nose.plugins.manager
+import yaml
+
 from paste import httpserver
 from six.moves import (
     http_client,
diff --git a/test/integration/test_realtime_tools_api.py b/test/integration/test_realtime_tools_api.py
index 6c6fc2d585d..645c032c6bd 100644
--- a/test/integration/test_realtime_tools_api.py
+++ b/test/integration/test_realtime_tools_api.py
@@ -3,16 +3,14 @@
 import os
 
 import requests
-
 from base import api_asserts
-
-from .test_containerized_jobs import ContainerizedIntegrationTestCase
-
 from base.populators import (
     DatasetPopulator,
     wait_on,
 )
 
+from .test_containerized_jobs import ContainerizedIntegrationTestCase
+
 SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
 
 

From 2ed75af635dec067a13cfa1482fafc97e5ac2fdc Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Thu, 30 May 2019 21:31:21 -0400
Subject: [PATCH 076/114] Update driver_util.py

---
 test/base/driver_util.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/base/driver_util.py b/test/base/driver_util.py
index e98ff07fff1..920684a211a 100644
--- a/test/base/driver_util.py
+++ b/test/base/driver_util.py
@@ -20,7 +20,6 @@ import nose.core
 import nose.loader
 import nose.plugins.manager
 import yaml
-
 from paste import httpserver
 from six.moves import (
     http_client,

From d05a3c995b539bb6df4b4f2403bf2354e4ec9381 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 4 Jun 2019 08:20:30 -0400
Subject: [PATCH 077/114] Add container_montior_command to MockJobWrapper

---
 test/unit/jobs/test_command_factory.py | 3 +++
 test/unit/jobs/test_runner_local.py    | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/test/unit/jobs/test_command_factory.py b/test/unit/jobs/test_command_factory.py
index b5493c3698a..20d2a5c15d8 100644
--- a/test/unit/jobs/test_command_factory.py
+++ b/test/unit/jobs/test_command_factory.py
@@ -178,6 +178,9 @@ class MockJobWrapper(object):
     def get_command_line(self):
         return self.command_line
 
+    def container_montior_command(self, *args, **kwds):
+        return None
+
     @property
     def requires_setting_metadata(self):
         return self.metadata_line is not None
diff --git a/test/unit/jobs/test_runner_local.py b/test/unit/jobs/test_runner_local.py
index 610da86a66f..70ab8113d8c 100644
--- a/test/unit/jobs/test_runner_local.py
+++ b/test/unit/jobs/test_runner_local.py
@@ -175,6 +175,9 @@ class MockJobWrapper(object):
     def get_command_line(self):
         return self.command_line
 
+    def container_montior_command(self, *args, **kwds):
+        return None
+
     def get_id_tag(self):
         return "1"
 

From 178c1ce54d5e73f2e4aaa7f9f479c1055947f349 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 4 Jun 2019 08:50:39 -0400
Subject: [PATCH 078/114] montior --> monitor

---
 lib/galaxy/jobs/__init__.py            | 2 +-
 lib/galaxy/jobs/command_factory.py     | 6 +++---
 test/unit/jobs/test_command_factory.py | 2 +-
 test/unit/jobs/test_runner_local.py    | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py
index 8a5bffcc08b..07070c01411 100644
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -1991,7 +1991,7 @@ class JobWrapper(HasResourceParameters):
             self.app.realtime_manager.configure_entry_points(job, container_runtime)
             return True
 
-    def container_montior_command(self, container, **kwds):
+    def container_monitor_command(self, container, **kwds):
         if not container or not self.tool.produces_entry_points:
             return None
 
diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py
index ef1701ba652..a93e1a6f0d5 100644
--- a/lib/galaxy/jobs/command_factory.py
+++ b/lib/galaxy/jobs/command_factory.py
@@ -103,9 +103,9 @@ def build_command(
         # xref https://github.com/galaxyproject/galaxy/issues/3289
         commands_builder.prepend_command("rm -rf working; mkdir -p working; cd working")
 
-    container_montior_command = job_wrapper.container_montior_command(container)
-    if container_montior_command:
-        commands_builder.prepend_command(container_montior_command)
+    container_monitor_command = job_wrapper.container_monitor_command(container)
+    if container_monitor_command:
+        commands_builder.prepend_command(container_monitor_command)
 
     if include_work_dir_outputs:
         __handle_work_dir_outputs(commands_builder, job_wrapper, runner, remote_command_params)
diff --git a/test/unit/jobs/test_command_factory.py b/test/unit/jobs/test_command_factory.py
index 20d2a5c15d8..66f47f6aae0 100644
--- a/test/unit/jobs/test_command_factory.py
+++ b/test/unit/jobs/test_command_factory.py
@@ -178,7 +178,7 @@ class MockJobWrapper(object):
     def get_command_line(self):
         return self.command_line
 
-    def container_montior_command(self, *args, **kwds):
+    def container_monitor_command(self, *args, **kwds):
         return None
 
     @property
diff --git a/test/unit/jobs/test_runner_local.py b/test/unit/jobs/test_runner_local.py
index 70ab8113d8c..afda49e09fc 100644
--- a/test/unit/jobs/test_runner_local.py
+++ b/test/unit/jobs/test_runner_local.py
@@ -175,7 +175,7 @@ class MockJobWrapper(object):
     def get_command_line(self):
         return self.command_line
 
-    def container_montior_command(self, *args, **kwds):
+    def container_monitor_command(self, *args, **kwds):
         return None
 
     def get_id_tag(self):

From 7b4c1d7675775e19ccb75edfb7c9f60b94abd1bb Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Thu, 6 Jun 2019 17:23:00 -0400
Subject: [PATCH 079/114] juypter --> jupyter

---
 test/functional/tools/realtimetool_juypter_notebook.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/functional/tools/realtimetool_juypter_notebook.xml b/test/functional/tools/realtimetool_juypter_notebook.xml
index f8c4f486b55..bc280eceb35 100644
--- a/test/functional/tools/realtimetool_juypter_notebook.xml
+++ b/test/functional/tools/realtimetool_juypter_notebook.xml
@@ -1,9 +1,9 @@
-<tool id="interactive_tool_juypter_notebook" tool_type="realtime" name="Interactive Jupyter Notebook" version="0.1">
+<tool id="interactive_tool_jupyter_notebook" tool_type="realtime" name="Interactive Jupyter Notebook" version="0.1">
     <requirements>
         <container type="docker">bgruening/docker-jupyter-notebook:ie2</container>
     </requirements>
     <realtime>
-        <entry_point name="Jypter Interactive Environment">
+        <entry_point name="Jupyter Interactive Tool">
             <port>8888</port>
             <url>ipython/tree</url>
         </entry_point>

From 4f347e072128e8ee27ba36dfbe8828e9b66fe054 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 5 Jul 2019 17:06:33 +0200
Subject: [PATCH 080/114] Change RealTimeTool to InteractiveTool in UI.

---
 .../components/ToolEntryPoints/ToolEntryPoints.vue     | 10 +++++-----
 client/galaxy/scripts/layout/menu.js                   |  2 +-
 lib/galaxy/webapps/galaxy/controllers/realtime.py      |  6 +++---
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue b/client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue
index 8e74d066724..af3a4e961fd 100644
--- a/client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue
+++ b/client/galaxy/scripts/components/ToolEntryPoints/ToolEntryPoints.vue
@@ -1,18 +1,18 @@
 <template>
     <div class="infomessagelarge">
         <p v-if="entryPoints.length==0" >
-            Waiting for RealTimeTool result view(s) to become available.
+            Waiting for InteractiveTool result view(s) to become available.
         </p>
         <p v-else-if="entryPoints.length==1" >
             <span v-if="entryPoints[0].active" >
-                There is a RealTimeTool result view available, <a :href="entryPoints[0].target">click here to display</a>.
+                There is an InteractiveTool result view available, <a :href="entryPoints[0].target">click here to display</a>.
             </span>
             <span v-else>
-                There is a RealTimeTool result view available, waiting for view to become active...
+                There is an InteractiveTool result view available, waiting for view to become active...
             </span>
         </p>
         <p v-else>
-            There is are multiple RealTimeTool result views available:
+            There are multiple InteractiveTool result views available:
             <ul>
                 <li v-for="entryPoint of entryPoints" v-bind:key="entryPoint.id" >
                     {{ entryPoint.name }}
@@ -26,7 +26,7 @@
             </ul>
         </p>
 
-        You may also access all active RealTimeTools from the User menu.
+        You may also access all active InteractiveTools from the User menu.
     </div>
 </template>
 
diff --git a/client/galaxy/scripts/layout/menu.js b/client/galaxy/scripts/layout/menu.js
index 7591b151f39..e7bacebe9fa 100644
--- a/client/galaxy/scripts/layout/menu.js
+++ b/client/galaxy/scripts/layout/menu.js
@@ -305,7 +305,7 @@ const Collection = Backbone.Collection.extend({
                         divider: true,
                     },
                     {
-                        title: _l("Active RealTimeTools"),
+                        title: _l("Active InteractiveTools"),
                         url: "realtime_entry_points/list",
                         target: "__use_router__"
                     }
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index e9af1f50737..18b2ca3e18c 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -31,7 +31,7 @@ class EntryPointLinkColumn(grids.GridColumn):
 class RealTimeToolEntryPointListGrid(grids.Grid):
 
     use_panels = True
-    title = "Available RealTimeTools"
+    title = "Available InteractiveTools"
     model_class = model.RealTimeToolEntryPoint
     default_filter = {"name": "All"}
     default_sort_key = "-update_time"
@@ -93,10 +93,10 @@ class RealTime(BaseUIController):
                         else:
                             succeeded.append(ep)
                     if failed:
-                        message = 'Unable to stop %i RealTimeTools.' % (len(failed))
+                        message = 'Unable to stop %i InteractiveTools.' % (len(failed))
                         status = 'error'
                     if succeeded:
-                        message = 'Stopped %i RealTimeTools.' % (len(succeeded))
+                        message = 'Stopped %i InteractiveTools.' % (len(succeeded))
                         status = 'ok'
         if message and status:
             kwargs['message'] = message

From eb228762162090ee761081fe7ee7a60734f2e885 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Fri, 5 Jul 2019 18:03:08 +0200
Subject: [PATCH 081/114] Change realtime tagset to entry_points.

---
 lib/galaxy/tool_util/parser/xml.py                 |  4 ++--
 lib/galaxy/tool_util/parser/yaml.py                |  2 +-
 lib/galaxy/tool_util/xsd/galaxy.xsd                | 14 +++++++-------
 lib/galaxy/tools/__init__.py                       |  6 +++---
 test/functional/tools/realtimetool_cellxgene.xml   |  6 +++---
 .../tools/realtimetool_juypter_notebook.xml        |  6 +++---
 test/functional/tools/realtimetool_simple.xml      |  6 +++---
 .../tools/realtimetool_two_entry_points.xml        |  6 +++---
 8 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py
index 886bdbdbbac..494768d474f 100644
--- a/lib/galaxy/tool_util/parser/xml.py
+++ b/lib/galaxy/tool_util/parser/xml.py
@@ -202,13 +202,13 @@ class XmlToolSource(ToolSource):
         return parallelism_info
 
     def parse_realtime(self):
-        realtimetool_el = self.root.find("realtime")
+        realtimetool_el = self.root.find("entry_points")
         rtt = []
         if realtimetool_el is None:
             return rtt
         for ep_el in realtimetool_el.findall("entry_point"):
             port = ep_el.find("port")
-            assert port is not None, ValueError('A port is required for RealTimeTools')
+            assert port is not None, ValueError('A port is required for InteractiveTools')
             port = port.text.strip()
             url = ep_el.find("url")
             if url is not None:
diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py
index 0216461aa6c..6a494d2436b 100644
--- a/lib/galaxy/tool_util/parser/yaml.py
+++ b/lib/galaxy/tool_util/parser/yaml.py
@@ -169,7 +169,7 @@ class YamlToolSource(ToolSource):
         return self.root_dict.get("profile", "16.04")
 
     def parse_realtime(self):
-        return self.root_dict.get("realtime", [])
+        return self.root_dict.get("entry_points", [])
 
     def parse_python_template_version(self):
         python_template_version = self.root_dict.get("python_template_version", None)
diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index ff768bf48b0..22e1ed09813 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -49,7 +49,7 @@ A ``data_source`` tool contains a few more relevant attributes.
         <xs:element name="edam_operations" type="EdamOperations" minOccurs="0"/>
         <xs:element name="xrefs" type="xrefs" minOccurs="0" />
         <xs:element name="requirements" type="Requirements" minOccurs="0"/>
-        <xs:element name="realtime" type="RealTime" minOccurs="0" maxOccurs="1" />
+        <xs:element name="entry_points" type="EntryPoints" minOccurs="0" maxOccurs="1" />
         <xs:element name="description" type="xs:string" minOccurs="0">
           <xs:annotation gxdocs:best_practices="tool-descriptions">
             <xs:documentation xml:lang="en"><![CDATA[The value is displayed in
@@ -226,12 +226,12 @@ For more information, see https://planemo.readthedocs.io/en/latest/writing_advan
     </xs:sequence>
   </xs:complexType>
 
-  <xs:complexType name="RealTime">
+  <xs:complexType name="EntryPoints">
     <xs:annotation>
       <xs:documentation xml:lang="en"><![CDATA[
 
 This is a container tag set for the ``entry_point`` tag that contains ``port`` and ``url`` tags
-described in greater detail below. ``entry_point``s describe RealTimeTool entry points
+described in greater detail below. ``entry_point``s describe InteractiveTool entry points
 to a tool.
 
 ]]></xs:documentation>
@@ -246,16 +246,16 @@ to a tool.
       <xs:documentation xml:lang="en"><![CDATA[
 
 This tag set is contained within the ``<entry_point>`` tag set. Access to entry point
-ports and urls are included in this tag set. These are used by RealTimeTools
+ports and urls are included in this tag set. These are used by InteractiveTools
 to provide access to graphical tools in real-time.
 
 ```xml
-<realtime>
+<entry_points>
   <entry_point name="Example name">
     <port>80</port>
     <url>landing/${template_enabled}/index.html</url>
   </entry_point>
-</realtime>
+</entry_points>
 ```
 
 ]]></xs:documentation>
@@ -5417,7 +5417,7 @@ and ``bibtex`` are the only supported options.</xs:documentation>
     <xs:restriction base="xs:string">
       <xs:enumeration value="data_source"/>
       <xs:enumeration value="manage_data"/>
-      <xs:enumeration value="realtime"/>
+      <xs:enumeration value="entry_points"/>
       <xs:enumeration value="expression"/>
     </xs:restriction>
   </xs:simpleType>
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index fddaa86c50f..9bfe6440533 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -535,7 +535,7 @@ class Tool(Dictifiable):
         # seem to require Galaxy's Python.
         # FIXME: the (instantiated) tool class should emit this behavior, and not
         #        use inspection by string check
-        if self.tool_type not in ["default", "manage_data", "realtime"]:
+        if self.tool_type not in ["default", "manage_data", "interactive"]:
             return True
 
         if self.tool_type == "manage_data" and self.profile < 18.09:
@@ -2431,7 +2431,7 @@ class ImportHistoryTool(Tool):
 
 
 class RealTimeTool(Tool):
-    tool_type = 'realtime'
+    tool_type = 'interactive'
     produces_entry_points = True
 
     def __remove_realtime_by_job(self, job):
@@ -2440,7 +2440,7 @@ class RealTimeTool(Tool):
             log.debug('__remove_realtime_by_job: %s', eps)
             self.app.realtime_manager.remove_entry_points(eps)
         else:
-            log.warning("Could not determine job to stop RealTimeTool: %s", job)
+            log.warning("Could not determine job to stop InteractiveTool: %s", job)
 
     def exec_after_process(self, app, inp_data, out_data, param_dict, job=None, **kwds):
         # run original exec_after_process
diff --git a/test/functional/tools/realtimetool_cellxgene.xml b/test/functional/tools/realtimetool_cellxgene.xml
index 43b56d24cb7..904f2292c24 100644
--- a/test/functional/tools/realtimetool_cellxgene.xml
+++ b/test/functional/tools/realtimetool_cellxgene.xml
@@ -1,12 +1,12 @@
-<tool id="interactive_tool_cellxgene" tool_type="realtime" name="Interactive CellXgene Environment" version="0.1">
+<tool id="interactive_tool_cellxgene" tool_type="interactive" name="Interactive CellXgene Environment" version="0.1">
     <requirements>
         <container type="docker">quay.io/galaxy/cellxgene-galaxy-ie:ie2</container>
     </requirements>
-    <realtime>
+    <entry_points>
         <entry_point name="cellxgene_single_cell_visualisation">
             <port>80</port>
         </entry_point>
-    </realtime>
+    </entry_points>
     <command><![CDATA[
         #import re
         #set $fancy_name = '/tmp/galaxy_cellxgene_' + re.sub('[^\w\-_]', '_', $input.element_identifier) + '.h5ad'
diff --git a/test/functional/tools/realtimetool_juypter_notebook.xml b/test/functional/tools/realtimetool_juypter_notebook.xml
index bc280eceb35..161a4c4b7ca 100644
--- a/test/functional/tools/realtimetool_juypter_notebook.xml
+++ b/test/functional/tools/realtimetool_juypter_notebook.xml
@@ -1,13 +1,13 @@
-<tool id="interactive_tool_jupyter_notebook" tool_type="realtime" name="Interactive Jupyter Notebook" version="0.1">
+<tool id="interactive_tool_jupyter_notebook" tool_type="interactive" name="Interactive Jupyter Notebook" version="0.1">
     <requirements>
         <container type="docker">bgruening/docker-jupyter-notebook:ie2</container>
     </requirements>
-    <realtime>
+    <entry_points>
         <entry_point name="Jupyter Interactive Tool">
             <port>8888</port>
             <url>ipython/tree</url>
         </entry_point>
-    </realtime>
+    </entry_points>
     <command detect_errors="aggressive"><![CDATA[
         #import re
         export GALAXY_WORKING_DIR=`pwd` &&
diff --git a/test/functional/tools/realtimetool_simple.xml b/test/functional/tools/realtimetool_simple.xml
index 5f20ec07c0f..9c6c48b9059 100644
--- a/test/functional/tools/realtimetool_simple.xml
+++ b/test/functional/tools/realtimetool_simple.xml
@@ -1,13 +1,13 @@
-<tool id="realtimetool_simple" name="realtimetool_simple" tool_type="realtime" version="0.1">
+<tool id="realtimetool_simple" name="realtimetool_simple" tool_type="interactive" version="0.1">
     <requirements>
         <container type="docker">galaxy/test-http-example:0.1</container>
     </requirements>
-    <realtime>
+    <entry_points>
         <entry_point name="Simple">
             <port>7000</port>
             <url>/</url>
         </entry_point>
-    </realtime>
+    </entry_points>
     <command detect_errors="exit_code"><![CDATA[
         cd /;
         python -m SimpleHTTPServer 7000
diff --git a/test/functional/tools/realtimetool_two_entry_points.xml b/test/functional/tools/realtimetool_two_entry_points.xml
index de5d439a4dc..dc45d9caa86 100644
--- a/test/functional/tools/realtimetool_two_entry_points.xml
+++ b/test/functional/tools/realtimetool_two_entry_points.xml
@@ -1,8 +1,8 @@
-<tool id="realtimetool_two_entry_points" name="realtimetool_two_entry_points" tool_type="realtime" version="0.1">
+<tool id="realtimetool_two_entry_points" name="realtimetool_two_entry_points" tool_type="interactive" version="0.1">
     <requirements>
         <container type="docker">galaxy/test-http-example:0.1</container>
     </requirements>
-    <realtime>
+    <entry_points>
         <entry_point name="Server 1">
             <port>7000</port>
             <url>/</url>
@@ -11,7 +11,7 @@
             <port>7001</port>
             <url>/</url>
         </entry_point>
-    </realtime>
+    </entry_points>
     <command detect_errors="exit_code"><![CDATA[
         cd /;
         python -m SimpleHTTPServer 7000 &

From 35055eda8e0aa5e6ba3bc31705da421715409d5f Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Sat, 6 Jul 2019 09:32:08 +0200
Subject: [PATCH 082/114] interactive

---
 lib/galaxy/tool_util/xsd/galaxy.xsd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index 22e1ed09813..5981582c726 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -5417,7 +5417,7 @@ and ``bibtex`` are the only supported options.</xs:documentation>
     <xs:restriction base="xs:string">
       <xs:enumeration value="data_source"/>
       <xs:enumeration value="manage_data"/>
-      <xs:enumeration value="entry_points"/>
+      <xs:enumeration value="interactive"/>
       <xs:enumeration value="expression"/>
     </xs:restriction>
   </xs:simpleType>

From 93015502f7b33b15df0d7be41ce1b220f5ae2891 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Sun, 7 Jul 2019 12:25:44 +0200
Subject: [PATCH 083/114] Fix for check_for_entry_points when file exists, but
 is not fully populated yet

---
 lib/galaxy/jobs/__init__.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py
index d154b097cbd..72efd1d82cd 100644
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -1977,7 +1977,11 @@ class JobWrapper(HasResourceParameters):
         container_runtime_path = os.path.join(working_directory, "container_runtime.json")
         if os.path.exists(container_runtime_path):
             with open(container_runtime_path, "r") as f:
-                container_runtime = json.load(f)
+                try:
+                    container_runtime = json.load(f)
+                except ValueError:
+                    # File exists, but is not fully populated yet
+                    return False
             log.debug("found container runtime %s" % container_runtime)
             self.app.realtime_manager.configure_entry_points(job, container_runtime)
             return True

From f0fce805f6fecc3bdcf6508ce68611616b8794c7 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Sun, 7 Jul 2019 17:20:24 +0200
Subject: [PATCH 084/114] Pass through tool defined environment_variables to
 Docker.

---
 lib/galaxy/tools/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 9bfe6440533..79589818155 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -697,6 +697,10 @@ class Tool(Dictifiable):
         self.home_target = home_target
         self.tmp_target = tmp_target
         self.docker_env_pass_through = tool_source.parse_docker_env_pass_through()
+        if self.environment_variables:
+            if not self.docker_env_pass_through:
+                self.docker_env_pass_through = []
+            self.docker_env_pass_through.extend(map(lambda x: x['name'], self.environment_variables))
 
         # Parameters used to build URL for redirection to external app
         redirect_url_params = tool_source.parse_redirect_url_params_elem()

From 102f93516175814a900ce8275bdecd643672c905 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Sun, 7 Jul 2019 18:10:42 +0200
Subject: [PATCH 085/114] from galaxy.web.base.controller import is now from
 galaxy.webapps.base.controller import

---
 lib/galaxy/webapps/galaxy/api/tool_entry_points.py | 2 +-
 lib/galaxy/webapps/galaxy/controllers/realtime.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/tool_entry_points.py b/lib/galaxy/webapps/galaxy/api/tool_entry_points.py
index 7b9dd0698fc..fc0cf9b8db5 100644
--- a/lib/galaxy/webapps/galaxy/api/tool_entry_points.py
+++ b/lib/galaxy/webapps/galaxy/api/tool_entry_points.py
@@ -6,7 +6,7 @@ import logging
 from galaxy import exceptions, util
 from galaxy.managers.realtime import RealTimeManager
 from galaxy.web import expose_api_anonymous_and_sessionless
-from galaxy.web.base.controller import BaseAPIController
+from galaxy.webapps.base.controller import BaseAPIController
 
 log = logging.getLogger(__name__)
 
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index 18b2ca3e18c..dda66a16ffb 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -7,7 +7,7 @@ from galaxy import (
     model,
     web
 )
-from galaxy.web.base.controller import (
+from galaxy.webapps.base.controller import (
     BaseUIController,
 )
 from galaxy.web.framework.helpers import (

From 75bc929693a56a1233236954b4e9e11d6a360442 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Sun, 7 Jul 2019 18:46:40 +0200
Subject: [PATCH 086/114] Add 404 when InteractiveTool route doesn't exist or
 can't be accessed

---
 config/galaxy.yml.realtime | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config/galaxy.yml.realtime b/config/galaxy.yml.realtime
index bb6f5203ff2..04ec6126aa9 100644
--- a/config/galaxy.yml.realtime
+++ b/config/galaxy.yml.realtime
@@ -19,6 +19,7 @@ uwsgi:
   route-label: realtime
   route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.localhost:8080)$ rpcvar:TARGET_HOST rtt_key_type_token_mapper_cached $2 $1 $3 $4 $0 5
   route-if-not: empty:${TARGET_HOST} httpdumb:${TARGET_HOST}
+  route: .* break:404 Not Found
   route-label: endendend
 
 

From 0d6a66f6a54c0fcaef4378bb789d059be97948b1 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Sun, 7 Jul 2019 19:15:10 +0200
Subject: [PATCH 087/114] Flake

---
 lib/galaxy/webapps/galaxy/controllers/realtime.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index dda66a16ffb..e43a92d1e60 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -7,13 +7,13 @@ from galaxy import (
     model,
     web
 )
-from galaxy.webapps.base.controller import (
-    BaseUIController,
-)
 from galaxy.web.framework.helpers import (
     grids,
     time_ago,
 )
+from galaxy.webapps.base.controller import (
+    BaseUIController,
+)
 
 log = logging.getLogger(__name__)
 

From 361ed34cd6ef045b60645d30d95e57064c3e3cd9 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Sun, 7 Jul 2019 19:57:10 +0200
Subject: [PATCH 088/114] Add ability to strip whitespace from tool
 environment_variables.

---
 lib/galaxy/tool_util/parser/xml.py  | 1 +
 lib/galaxy/tool_util/xsd/galaxy.xsd | 5 +++++
 lib/galaxy/tools/evaluation.py      | 6 ++++--
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py
index ed101a52c52..e3e9979b4c8 100644
--- a/lib/galaxy/tool_util/parser/xml.py
+++ b/lib/galaxy/tool_util/parser/xml.py
@@ -141,6 +141,7 @@ class XmlToolSource(ToolSource):
             definition = {
                 "name": environment_variable_el.get("name"),
                 "template": environment_variable_el.text,
+                "strip": string_as_bool(environment_variable_el.get("strip", False)),
             }
             environment_variables.append(
                 definition
diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index 5981582c726..7379cce952a 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -4647,6 +4647,11 @@ variable instead of shell variable.
 define.</xs:documentation>
           </xs:annotation>
         </xs:attribute>
+        <xs:attribute name="strip" type="PermissiveBoolean" default="false">
+        <xs:annotation>
+          <xs:documentation xml:lang="en">Whether to strip whitespace.</xs:documentation>
+        </xs:annotation>
+      </xs:attribute>
       </xs:extension>
     </xs:simpleContent>
   </xs:complexType>
diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py
index 12f875eea01..1d827a6b697 100644
--- a/lib/galaxy/tools/evaluation.py
+++ b/lib/galaxy/tools/evaluation.py
@@ -545,7 +545,7 @@ class ToolEvaluator(object):
             environment_variable_template = environment_variable_def["template"]
             fd, config_filename = tempfile.mkstemp(dir=directory)
             os.close(fd)
-            self.__write_workdir_file(config_filename, environment_variable_template, param_dict)
+            self.__write_workdir_file(config_filename, environment_variable_template, param_dict, strip=environment_variable_def["strip"])
             config_file_basename = os.path.basename(config_filename)
             environment_variable["value"] = "`cat %s`" % config_file_basename
             environment_variable["raw"] = True
@@ -599,11 +599,13 @@ class ToolEvaluator(object):
 
         return json.dumps(wrapped_json.json_wrap(self.tool.inputs, self.param_dict, handle_files=handle_files)), False
 
-    def __write_workdir_file(self, config_filename, content, context, is_template=True):
+    def __write_workdir_file(self, config_filename, content, context, is_template=True, strip=False):
         if is_template:
             value = fill_template(content, context=context, python_template_version=self.tool.python_template_version)
         else:
             value = unicodify(content)
+        if strip:
+            value = value.strip()
         with io.open(config_filename, "w", encoding='utf-8') as f:
             f.write(value)
         # For running jobs as the actual user, ensure the config file is globally readable

From fd7f0d3d8fee82727843bbd1665720e59d6611a6 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 8 Jul 2019 12:17:38 +0200
Subject: [PATCH 089/114] Fix for fetching strip if not defined

---
 lib/galaxy/tools/evaluation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py
index 1d827a6b697..3a30d6372df 100644
--- a/lib/galaxy/tools/evaluation.py
+++ b/lib/galaxy/tools/evaluation.py
@@ -545,7 +545,7 @@ class ToolEvaluator(object):
             environment_variable_template = environment_variable_def["template"]
             fd, config_filename = tempfile.mkstemp(dir=directory)
             os.close(fd)
-            self.__write_workdir_file(config_filename, environment_variable_template, param_dict, strip=environment_variable_def["strip"])
+            self.__write_workdir_file(config_filename, environment_variable_template, param_dict, strip=environment_variable_def.get("strip", False))
             config_file_basename = os.path.basename(config_filename)
             environment_variable["value"] = "`cat %s`" % config_file_basename
             environment_variable["raw"] = True

From e04e03d772dbe82eab9abf83c26562df84471760 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Thu, 25 Jul 2019 14:59:36 -0400
Subject: [PATCH 090/114] Don't set user in example interactive tool
 job_conf.xml

---
 config/job_conf.xml.realtime | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config/job_conf.xml.realtime b/config/job_conf.xml.realtime
index 8cf4ee48117..a42e3305ad8 100644
--- a/config/job_conf.xml.realtime
+++ b/config/job_conf.xml.realtime
@@ -14,6 +14,7 @@
           <param id="docker_net">bridge</param>
           <param id="docker_auto_rm">true</param>
           <param id="require_container">true</param>
+          <param id="docker_set_user"></param>
         </destination>
 
         <destination id="docker_dispatch" runner="dynamic">

From 04b1e8f79c7dd5df2b95ca31027eb9a302520f9c Mon Sep 17 00:00:00 2001
From: Bjoern Gruening <bjoern.gruening@gmail.com>
Date: Sun, 11 Aug 2019 16:41:00 +0200
Subject: [PATCH 091/114] add a bunch of ITs

---
 .../tools/interactive_tool_askomics.xml       | 39 ++++++++++++
 .../tools/interactive_tool_bam_iobio.xml      | 45 +++++++++++++
 ...ene.xml => interactive_tool_cellxgene.xml} |  8 +--
 .../tools/interactive_tool_ethercalc.xml      | 63 +++++++++++++++++++
 .../tools/interactive_tool_hicbrowser.xml     | 33 ++++++++++
 ... => interactive_tool_juypter_notebook.xml} |  0
 .../tools/interactive_tool_neo4j.xml          | 37 +++++++++++
 .../tools/interactive_tool_phinch.xml         | 37 +++++++++++
 ...simple.xml => interactive_tool_simple.xml} |  0
 ... => interactive_tool_two_entry_points.xml} |  0
 .../tools/realtimetool_ethercalc.xml          | 63 +++++++++++++++++++
 11 files changed, 321 insertions(+), 4 deletions(-)
 create mode 100644 test/functional/tools/interactive_tool_askomics.xml
 create mode 100644 test/functional/tools/interactive_tool_bam_iobio.xml
 rename test/functional/tools/{realtimetool_cellxgene.xml => interactive_tool_cellxgene.xml} (73%)
 create mode 100644 test/functional/tools/interactive_tool_ethercalc.xml
 create mode 100644 test/functional/tools/interactive_tool_hicbrowser.xml
 rename test/functional/tools/{realtimetool_juypter_notebook.xml => interactive_tool_juypter_notebook.xml} (100%)
 create mode 100644 test/functional/tools/interactive_tool_neo4j.xml
 create mode 100644 test/functional/tools/interactive_tool_phinch.xml
 rename test/functional/tools/{realtimetool_simple.xml => interactive_tool_simple.xml} (100%)
 rename test/functional/tools/{realtimetool_two_entry_points.xml => interactive_tool_two_entry_points.xml} (100%)
 create mode 100644 test/functional/tools/realtimetool_ethercalc.xml

diff --git a/test/functional/tools/interactive_tool_askomics.xml b/test/functional/tools/interactive_tool_askomics.xml
new file mode 100644
index 00000000000..ab48f0e6322
--- /dev/null
+++ b/test/functional/tools/interactive_tool_askomics.xml
@@ -0,0 +1,39 @@
+<tool id="interactive_tool_askomics" tool_type="interactive" name="AskOmics" version="0.1">
+    <description>AskOmics, a visual SPARQL query builder</description>
+    <requirements>
+        <container type="docker">askomics/askomics-ie</container>
+    </requirements>
+    <entry_points>
+        <entry_point name="AskOmics instance on $infile.display_name">
+            <port>6543</port>
+            <url>/login_api_gie?key=abcd</url>
+        </entry_point>
+    </entry_points>
+    <command><![CDATA[
+        ## ToDo: the key could be generated randomly
+        export ASKO_load_url='http://localhost:6543' &&
+        export ASKOMICS_API_KEY='abcd' &&
+        export ASKO_files_dir='/tmp/askomics-ie'  &&
+        export VIRT_Parameters_NumberOfBuffers='10000'  &&
+        export VIRT_Parameters_MaxDirtyBuffers='6000'  &&
+
+        mkdir -p /import &&
+        cp '$infile' /import/foo.bed &&
+
+        sed -i.bak 's@^/monitor_traffic.*@@g' /start.sh &&
+        bash /start.sh
+    ]]>
+    </command>
+    <inputs>
+        <param name="infile" type="data" format="tabular,gff,gff3,bed,interval" label="A datasets with genomic coordinates"/>
+    </inputs>
+    <outputs>
+        <data name="outfile" format="txt" />
+    </outputs>
+    <tests>
+    </tests>
+    <help>
+        AskOmics is a visual SPARQL query interface supporting both intuitive data integration and
+        querying while shielding the user from most of the technical difficulties underlying RDF and SPARQL.
+    </help>
+</tool>
diff --git a/test/functional/tools/interactive_tool_bam_iobio.xml b/test/functional/tools/interactive_tool_bam_iobio.xml
new file mode 100644
index 00000000000..ac32a842bfb
--- /dev/null
+++ b/test/functional/tools/interactive_tool_bam_iobio.xml
@@ -0,0 +1,45 @@
+<tool id="interactive_tool_bam_iobio" tool_type="interactive" name="BAM (iobio) Visualisation" version="0.1">
+    <requirements>
+        <container type="docker">qiaoy/iobio-bundle.bam-iobio:1.0-ondemand</container>
+    </requirements>
+    <entry_points>
+        <entry_point name="BAM io.bio visualisation of $infile.display_name">
+            <port>80</port>
+            <url><![CDATA[/?bam=http://localhost/tmp/bamfile.bam&region=1]]></url>
+        </entry_point>
+    </entry_points>
+    <command><![CDATA[
+    ## ToDo: websocket could not be found
+    ## WebSocket connection to 'ws://localhost/bamreaddepther/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED
+
+    #set $PUB_HOSTNAME = 'localhost'
+    #set $PUB_HTTP_PORT = '80'
+
+    cd /var/www/html &&
+    sed -i "s@wss://services.iobio.io/samtools/@ws://127.0.0.1/samtools/@" js/bam.iobio.js/bam.iobio.js &&
+    sed -i "s@wss://services.iobio.io/bamreaddepther/@ws://${PUB_HOSTNAME}:${PUB_HTTP_PORT}/bamreaddepther/@" js/bam.iobio.js/bam.iobio.js &&
+    sed -i "s@wss://services.iobio.io/bamstatsalive/@ws://${PUB_HOSTNAME}:${PUB_HTTP_PORT}/bamstatsalive/@" js/bam.iobio.js/bam.iobio.js &&
+    sed -i "s@wss://services.iobio.io/samheader/@ws://${PUB_HOSTNAME}:${PUB_HTTP_PORT}/samheader/@" js/bam.iobio.js/bam.iobio.js &&
+
+
+        cp '${infile}' /input/bamfile.bam &&
+        cp '${infile.metadata.bam_index}' /input/bamfile.bam.bai &&
+        mkdir /var/log/supervisor/ &&
+        head -n -2  /etc/supervisor.d/app.conf > /tmp/app.conf &&
+        mv /tmp/app.conf /etc/supervisor.d/app.conf &&
+
+        /usr/bin/supervisord -c /etc/supervisord.conf
+    ]]>
+    </command>
+    <inputs>
+        <param name="infile" type="data" format="bam" label="BAM file"/>
+    </inputs>
+    <outputs>
+        <data name="outfile" format="txt" />
+    </outputs>
+    <tests>
+    </tests>
+    <help>
+        BAM iobio visualisation.
+    </help>
+</tool>
diff --git a/test/functional/tools/realtimetool_cellxgene.xml b/test/functional/tools/interactive_tool_cellxgene.xml
similarity index 73%
rename from test/functional/tools/realtimetool_cellxgene.xml
rename to test/functional/tools/interactive_tool_cellxgene.xml
index 904f2292c24..0eb659681e5 100644
--- a/test/functional/tools/realtimetool_cellxgene.xml
+++ b/test/functional/tools/interactive_tool_cellxgene.xml
@@ -3,20 +3,20 @@
         <container type="docker">quay.io/galaxy/cellxgene-galaxy-ie:ie2</container>
     </requirements>
     <entry_points>
-        <entry_point name="cellxgene_single_cell_visualisation">
+        <entry_point name="Cellxgene Single Cell Visualisation on $infile.display_name">
             <port>80</port>
         </entry_point>
     </entry_points>
     <command><![CDATA[
         #import re
-        #set $fancy_name = '/tmp/galaxy_cellxgene_' + re.sub('[^\w\-_]', '_', $input.element_identifier) + '.h5ad'
-        cp '${input}' '${fancy_name}'
+        #set $fancy_name = '/tmp/galaxy_cellxgene_' + re.sub('[^\w\-_]', '_', $infile.element_identifier) + '.h5ad'
+        cp '${infile}' '${fancy_name}'
         &&
         cellxgene launch --host 0.0.0.0 --port 80 '${fancy_name}'
     ]]>
     </command>
     <inputs>
-        <param name="input" type="data" format="h5ad" label="Concatenate Dataset"/>
+        <param name="infile" type="data" format="h5ad" label="Concatenate Dataset"/>
     </inputs>
     <outputs>
         <data name="out_file1" format="txt" />
diff --git a/test/functional/tools/interactive_tool_ethercalc.xml b/test/functional/tools/interactive_tool_ethercalc.xml
new file mode 100644
index 00000000000..db37831f6cd
--- /dev/null
+++ b/test/functional/tools/interactive_tool_ethercalc.xml
@@ -0,0 +1,63 @@
+<tool id="interactive_tool_ethercalc" tool_type="interactive" name="EtherCalc" version="0.1">
+    <requirements>
+        <container type="docker">shiltemann/ethercalc-galaxy-ie:17.05</container>
+    </requirements>
+    <entry_points>
+        <entry_point name="EtherCalc instance on $infile.display_name">
+            <port>8000</port>
+        </entry_point>
+    </entry_points>
+    <command><![CDATA[
+    
+        forever start `which ethercalc` --cors &
+        bash $wait_script
+        &&
+
+        curl --include --request POST --header "Content-Type: application/json" --data-binary "{ \"room\": \"galaxy\", \"snapshot\": \"...\"}" 'http://localhost:8000/_'
+        &&
+
+        ## put loading message into worksheet in case of large files
+        echo -e "Loading your,dataset\nThis may take, some time for, large files.." > loading.txt
+        &&
+        curl --include --request PUT --header "Content-Type: text/csv" --data-binary @loading.txt http://localhost:8000/_/galaxy
+        &&
+
+        ## remove dump file so this doesnt appear in audit trail
+        rm /dump.json
+        &&
+
+        ## load dataset into worksheet
+        curl --include --request PUT --header "Content-Type: text/csv" --data-binary @$infile http://localhost:8000/_/galaxy
+        &&
+
+        tail -f /etc/hosts
+
+    ]]>
+    </command>
+    <configfiles>
+        <configfile name="wait_script">
+<![CDATA[
+## make sure ethercalc has finished starting up
+STATUS=\$(curl --include 'http://localhost:8000/_/galaxy' 2>&1)
+while [[ \${STATUS} =~ "refused" ]]
+do
+  echo "waiting for ethercalc: \$STATUS \n"
+  STATUS=\$(curl --include 'http://localhost:8000/_/galaxy' 2>&1)
+  sleep 2
+done
+    ]]>
+        </configfile>
+    </configfiles>
+    <inputs>
+        <param name="infile" type="data" format="tabular,csv,tsv" label="Some tabular dataset"/>
+    </inputs>
+    <outputs>
+        <data name="outfile" format="txt" />
+    </outputs>
+    <tests>
+    </tests>
+    <help>
+        EtherCalc is a web spreadsheet.
+        https://ethercalc.net
+    </help>
+</tool>
diff --git a/test/functional/tools/interactive_tool_hicbrowser.xml b/test/functional/tools/interactive_tool_hicbrowser.xml
new file mode 100644
index 00000000000..aed7ef765af
--- /dev/null
+++ b/test/functional/tools/interactive_tool_hicbrowser.xml
@@ -0,0 +1,33 @@
+<tool id="interactive_tool_hicbrowser" tool_type="interactive" name="HiCBrowser" version="0.1">
+    <requirements>
+        <container type="docker">bgruening/hicbrowser</container>
+    </requirements>
+    <entry_points>
+        <entry_point name="HiCBrowser on $infile.display_name">
+            <port>80</port>
+        </entry_point>
+    </entry_points>
+    <command><![CDATA[
+        #set $PROXY_PREFIX = ""
+        sed -i "s|PROXY_PREFIX|${PROXY_PREFIX}|" /etc/nginx/conf.d/nginx.conf &&
+
+        cp '$infile' /data/data_pack.tar.gz &&
+        cd /data &&
+        tar xvfz data_pack.tar.gz &&
+        supervisord &&
+        tail -f /var/log/supervisor/*
+
+    ]]>
+    </command>
+    <inputs>
+        <param name="infile" type="data" format="tar.gz,zip" label="HiCExplorer archive"/>
+    </inputs>
+    <outputs>
+        <data name="outfile" format="txt" />
+    </outputs>
+    <tests>
+    </tests>
+    <help>
+        Visualising HiC data with HiCBrowser.
+    </help>
+</tool>
diff --git a/test/functional/tools/realtimetool_juypter_notebook.xml b/test/functional/tools/interactive_tool_juypter_notebook.xml
similarity index 100%
rename from test/functional/tools/realtimetool_juypter_notebook.xml
rename to test/functional/tools/interactive_tool_juypter_notebook.xml
diff --git a/test/functional/tools/interactive_tool_neo4j.xml b/test/functional/tools/interactive_tool_neo4j.xml
new file mode 100644
index 00000000000..2dae5239ccb
--- /dev/null
+++ b/test/functional/tools/interactive_tool_neo4j.xml
@@ -0,0 +1,37 @@
+<tool id="interactive_tool_neo4j" tool_type="interactive" name="Neo4j (Graph Database)" version="0.1">
+    <requirements>
+        <container type="docker">quay.io/sanbi-sa/neo_ie:3.1.9</container>
+    </requirements>
+    <entry_points>
+        <entry_point name="Neo4j on $infile.display_name">
+            <port>80</port>
+        </entry_point>
+        <!-- I get some web-socket error that 7687 is not reachable. 
+            but docker does not expose 7687?
+        -->
+    </entry_points>
+    <command><![CDATA[
+    
+        export USER_UID=2345 &&
+        export USER_GID=2345 &&
+        export MONITOR_TRAFFIC=false &&
+        mkdir -p /data/neo4jdb &&
+        unzip '$infile' -d /data/neo4jdb &&
+        ls -l /data/neo4jdb &&
+        sed -i.bak 's@/monitor_traffic.*@@g' /docker-entrypoint.sh &&
+        cd /opt/neo4j &&
+        bash /docker-entrypoint.sh neo4j
+    ]]>
+    </command>
+    <inputs>
+        <param name="infile" type="data" format="zip" label="Neo4J data store"/>
+    </inputs>
+    <outputs>
+        <data name="outfile" format="txt" />
+    </outputs>
+    <tests>
+    </tests>
+    <help>
+        Neo4j is a highly scalable, robust native graph database.
+    </help>
+</tool>
diff --git a/test/functional/tools/interactive_tool_phinch.xml b/test/functional/tools/interactive_tool_phinch.xml
new file mode 100644
index 00000000000..e7c6706be5f
--- /dev/null
+++ b/test/functional/tools/interactive_tool_phinch.xml
@@ -0,0 +1,37 @@
+<tool id="interactive_tool_pinch" tool_type="interactive" name="Phinch Visualisation" version="0.1">
+    <requirements>
+        <container type="docker">shiltemann/docker-phinch-galaxy:16.04</container>
+    </requirements>
+    <entry_points>
+        <entry_point name="Phinch Visualisation of $infile.display_name">
+            <port>80</port>
+        </entry_point>
+    </entry_points>
+    <command><![CDATA[
+        ## ToDo nginx, proxy.conf etc can be removed from the container
+
+        #import os
+        #set $name = os.path.splitext(str($infile.display_name))[0]
+        ## in case someone names the data testdata.
+        rm /home/Phinch/data/testdata.biom | true &&
+        ln -s '$infile' /home/Phinch/data/${name}.biom &&
+        cd /home/Phinch/data &&
+        sed -i  "s/'REPLACE_ME'/'${name}.biom'/g" /home/Phinch/scripts/readFile.js &&
+        ## keep it running
+        cd /home/Phinch &&
+        php -S 0.0.0.0:80 2>&1 > /var/log/phinch.log
+
+    ]]>
+    </command>
+    <inputs>
+        <param name="infile" type="data" format="biom1" label="Biom1 dataset"/>
+    </inputs>
+    <outputs>
+        <data name="outfile" format="txt" />
+    </outputs>
+    <tests>
+    </tests>
+    <help>
+        Interactive tool for visualising Biom data.
+    </help>
+</tool>
diff --git a/test/functional/tools/realtimetool_simple.xml b/test/functional/tools/interactive_tool_simple.xml
similarity index 100%
rename from test/functional/tools/realtimetool_simple.xml
rename to test/functional/tools/interactive_tool_simple.xml
diff --git a/test/functional/tools/realtimetool_two_entry_points.xml b/test/functional/tools/interactive_tool_two_entry_points.xml
similarity index 100%
rename from test/functional/tools/realtimetool_two_entry_points.xml
rename to test/functional/tools/interactive_tool_two_entry_points.xml
diff --git a/test/functional/tools/realtimetool_ethercalc.xml b/test/functional/tools/realtimetool_ethercalc.xml
new file mode 100644
index 00000000000..387e931e405
--- /dev/null
+++ b/test/functional/tools/realtimetool_ethercalc.xml
@@ -0,0 +1,63 @@
+<tool id="interactive_tool_ethercalc" tool_type="interactive" name="EtherCalc" version="0.1">
+    <requirements>
+        <container type="docker">shiltemann/ethercalc-galaxy-ie:17.05</container>
+    </requirements>
+    <entry_points>
+        <entry_point name="ethercalc">
+            <port>8000</port>
+        </entry_point>
+    </entry_points>
+    <command><![CDATA[
+    
+        forever start `which ethercalc` --cors &
+
+        ## make sure ethercalc has finished starting up
+        STATUS=$(curl --include 'http://localhost:8000/_/galaxy' 2>&1)
+        while [[ ${STATUS} =~ "refused" ]]
+        do
+          echo "waiting for ethercalc: $STATUS \n"
+          STATUS=$(curl --include 'http://localhost:8000/_/galaxy' 2>&1)
+          sleep 2
+        done
+        &&
+
+
+        curl --include \
+             --request POST \
+             --header "Content-Type: application/json" \
+             --data-binary "{ \"room\": \"galaxy\", \"snapshot\": \"...\"}"  \
+        'http://localhost:8000/_'
+        &&
+
+        ## put loading message into worksheet in case of large files
+        echo -e "Loading your,dataset\nThis may take, some time for, large files.." > loading.txt
+        curl --include --request PUT --header "Content-Type: text/csv" --data-binary @loading.txt http://localhost:8000/_/galaxy
+
+        &&
+        ## remove dump file so this doesnt appear in audit trail
+        rm /dump.json
+        &&
+
+        ## load dataset into worksheet
+        curl --include --request PUT \
+        --header "Content-Type: text/csv" \
+        --data-binary $infile http://localhost:8000/_/galaxy
+        &&
+
+        tail -f /etc/hosts
+
+    ]]>
+    </command>
+    <inputs>
+        <param name="infile" type="data" format="tabular,csv,tsv" label="Some tabular dataset"/>
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="txt" />
+    </outputs>
+    <tests>
+    </tests>
+    <help>
+        EtherCalc is a web spreadsheet.
+        https://ethercalc.net
+    </help>
+</tool>

From 3689d857df0ba03b8c8e6548b16ffdd8f6f9723b Mon Sep 17 00:00:00 2001
From: Bjoern Gruening <bjoern.gruening@gmail.com>
Date: Sun, 11 Aug 2019 21:28:04 +0200
Subject: [PATCH 092/114] some small changes

---
 .../tools/interactive_tool_hicbrowser.xml           |  8 ++++----
 .../tools/interactive_tool_juypter_notebook.xml     | 13 +++++++++++++
 test/functional/tools/interactive_tool_neo4j.xml    | 12 ++++++++----
 3 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/test/functional/tools/interactive_tool_hicbrowser.xml b/test/functional/tools/interactive_tool_hicbrowser.xml
index aed7ef765af..3d50401ec5f 100644
--- a/test/functional/tools/interactive_tool_hicbrowser.xml
+++ b/test/functional/tools/interactive_tool_hicbrowser.xml
@@ -10,17 +10,17 @@
     <command><![CDATA[
         #set $PROXY_PREFIX = ""
         sed -i "s|PROXY_PREFIX|${PROXY_PREFIX}|" /etc/nginx/conf.d/nginx.conf &&
-
-        cp '$infile' /data/data_pack.tar.gz &&
+        rm /data/* -rf &&
+        cp '$infile' /data/data_pack.zip &&
         cd /data &&
-        tar xvfz data_pack.tar.gz &&
+        unzip data_pack.zip &&
         supervisord &&
         tail -f /var/log/supervisor/*
 
     ]]>
     </command>
     <inputs>
-        <param name="infile" type="data" format="tar.gz,zip" label="HiCExplorer archive"/>
+        <param name="infile" type="data" format="zip" label="HiCExplorer archive"/>
     </inputs>
     <outputs>
         <data name="outfile" format="txt" />
diff --git a/test/functional/tools/interactive_tool_juypter_notebook.xml b/test/functional/tools/interactive_tool_juypter_notebook.xml
index 161a4c4b7ca..d7f27022d2d 100644
--- a/test/functional/tools/interactive_tool_juypter_notebook.xml
+++ b/test/functional/tools/interactive_tool_juypter_notebook.xml
@@ -2,13 +2,26 @@
     <requirements>
         <container type="docker">bgruening/docker-jupyter-notebook:ie2</container>
     </requirements>
+
+
+HISTORY_ID
+
+
     <entry_points>
         <entry_point name="Jupyter Interactive Tool">
             <port>8888</port>
             <url>ipython/tree</url>
         </entry_point>
     </entry_points>
+    <environment_variables>
+        <environment_variable name="HISTORY_ID">${__app__.security.encode_id($jupyter_notebook.history_id)}</environment_variable>
+        <environment_variable name="REMOTE_HOST">TODO</environment_variable>
+        <environment_variable name="GALAXY_URL">TODO</environment_variable>
+        <environment_variable name="API_KEY">TODO</environment_variable>
+    </environment_variables>
+
     <command detect_errors="aggressive"><![CDATA[
+    
         #import re
         export GALAXY_WORKING_DIR=`pwd` &&
         mkdir -p ./jupyter/outputs/ &&
diff --git a/test/functional/tools/interactive_tool_neo4j.xml b/test/functional/tools/interactive_tool_neo4j.xml
index 2dae5239ccb..d1e21dec82a 100644
--- a/test/functional/tools/interactive_tool_neo4j.xml
+++ b/test/functional/tools/interactive_tool_neo4j.xml
@@ -10,11 +10,15 @@
             but docker does not expose 7687?
         -->
     </entry_points>
+    <environment_variables>
+        <environment_variable name="USER_GID">2345</environment_variable>
+        <environment_variable name="USER_UID">2345</environment_variable>
+        <environment_variable name="MONITOR_TRAFFIC">false</environment_variable>
+    </environment_variables>
     <command><![CDATA[
-    
-        export USER_UID=2345 &&
-        export USER_GID=2345 &&
-        export MONITOR_TRAFFIC=false &&
+        ##export USER_UID=2345 &&
+        ##export USER_GID=2345 &&
+        ##export MONITOR_TRAFFIC=false &&
         mkdir -p /data/neo4jdb &&
         unzip '$infile' -d /data/neo4jdb &&
         ls -l /data/neo4jdb &&

From 29e1173f6c4db870ace6b22234ad0795da066f0f Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 12 Aug 2019 12:23:38 -0400
Subject: [PATCH 093/114] Add API_KEY, GALAXY_URL, and fix HISTORY_ID envars

in interactive_tool_juypter_notebook.xml
---
 .../interactive_tool_juypter_notebook.xml     | 26 ++++++++++++-------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/test/functional/tools/interactive_tool_juypter_notebook.xml b/test/functional/tools/interactive_tool_juypter_notebook.xml
index d7f27022d2d..833c6c8264a 100644
--- a/test/functional/tools/interactive_tool_juypter_notebook.xml
+++ b/test/functional/tools/interactive_tool_juypter_notebook.xml
@@ -2,11 +2,6 @@
     <requirements>
         <container type="docker">bgruening/docker-jupyter-notebook:ie2</container>
     </requirements>
-
-
-HISTORY_ID
-
-
     <entry_points>
         <entry_point name="Jupyter Interactive Tool">
             <port>8888</port>
@@ -14,14 +9,25 @@ HISTORY_ID
         </entry_point>
     </entry_points>
     <environment_variables>
-        <environment_variable name="HISTORY_ID">${__app__.security.encode_id($jupyter_notebook.history_id)}</environment_variable>
+        <environment_variable name="HISTORY_ID" strip="True">
+            #if $jupyter_notebook:
+                ${__app__.security.encode_id($jupyter_notebook.history_id)}
+            #else:
+                ${__app__.security.encode_id($out_file1.history_id)}
+            #end if
+        </environment_variable>
         <environment_variable name="REMOTE_HOST">TODO</environment_variable>
-        <environment_variable name="GALAXY_URL">TODO</environment_variable>
-        <environment_variable name="API_KEY">TODO</environment_variable>
+        <environment_variable name="GALAXY_URL">${__app__.config.galaxy_infrastructure_url}</environment_variable>
+        <environment_variable name="API_KEY" strip="True">
+            #if $__user__:
+                #for $api_key in $__user__.api_keys:
+                    ${api_key.key}
+                    #break
+                #end for
+            #end if
+        </environment_variable>
     </environment_variables>
-
     <command detect_errors="aggressive"><![CDATA[
-    
         #import re
         export GALAXY_WORKING_DIR=`pwd` &&
         mkdir -p ./jupyter/outputs/ &&

From 5655ace0836506bfe864698cbc464b23adc7df7d Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Mon, 12 Aug 2019 15:41:23 -0400
Subject: [PATCH 094/114] Fix websockets in interactive_tool_bam_iobio.xml

---
 test/functional/tools/interactive_tool_bam_iobio.xml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/test/functional/tools/interactive_tool_bam_iobio.xml b/test/functional/tools/interactive_tool_bam_iobio.xml
index ac32a842bfb..a2006e5a5ee 100644
--- a/test/functional/tools/interactive_tool_bam_iobio.xml
+++ b/test/functional/tools/interactive_tool_bam_iobio.xml
@@ -16,11 +16,10 @@
     #set $PUB_HTTP_PORT = '80'
 
     cd /var/www/html &&
-    sed -i "s@wss://services.iobio.io/samtools/@ws://127.0.0.1/samtools/@" js/bam.iobio.js/bam.iobio.js &&
-    sed -i "s@wss://services.iobio.io/bamreaddepther/@ws://${PUB_HOSTNAME}:${PUB_HTTP_PORT}/bamreaddepther/@" js/bam.iobio.js/bam.iobio.js &&
-    sed -i "s@wss://services.iobio.io/bamstatsalive/@ws://${PUB_HOSTNAME}:${PUB_HTTP_PORT}/bamstatsalive/@" js/bam.iobio.js/bam.iobio.js &&
-    sed -i "s@wss://services.iobio.io/samheader/@ws://${PUB_HOSTNAME}:${PUB_HTTP_PORT}/samheader/@" js/bam.iobio.js/bam.iobio.js &&
-
+    sed -i "s@\"wss://services.iobio.io/samtools/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/samtools/\"@" js/bam.iobio.js/bam.iobio.js &&
+    sed -i "s@\"wss://services.iobio.io/bamreaddepther/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/bamreaddepther/\"@" js/bam.iobio.js/bam.iobio.js &&
+    sed -i "s@\"wss://services.iobio.io/bamstatsalive/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/bamstatsalive/\"@" js/bam.iobio.js/bam.iobio.js &&
+    sed -i "s@\"wss://services.iobio.io/samheader/\"@((window.location.protocol === \"https:\") ? \"wss://\" : \"ws://\") + window.location.host + \"/samheader/\"@" js/bam.iobio.js/bam.iobio.js &&
 
         cp '${infile}' /input/bamfile.bam &&
         cp '${infile.metadata.bam_index}' /input/bamfile.bam.bai &&

From 22edfcc11eab594315a3c1555489e7928534ac0e Mon Sep 17 00:00:00 2001
From: Anthony Bretaudeau <anthony.bretaudeau@inria.fr>
Date: Tue, 13 Aug 2019 15:07:11 +0200
Subject: [PATCH 095/114] fixes for askomics IT

---
 .../tools/interactive_tool_askomics.xml       | 26 ++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/test/functional/tools/interactive_tool_askomics.xml b/test/functional/tools/interactive_tool_askomics.xml
index ab48f0e6322..3014321a164 100644
--- a/test/functional/tools/interactive_tool_askomics.xml
+++ b/test/functional/tools/interactive_tool_askomics.xml
@@ -1,7 +1,7 @@
 <tool id="interactive_tool_askomics" tool_type="interactive" name="AskOmics" version="0.1">
     <description>AskOmics, a visual SPARQL query builder</description>
     <requirements>
-        <container type="docker">askomics/askomics-ie</container>
+        <container type="docker">quay.io/askomics/askomics-ie:17.12_g19.09</container>
     </requirements>
     <entry_points>
         <entry_point name="AskOmics instance on $infile.display_name">
@@ -9,7 +9,19 @@
             <url>/login_api_gie?key=abcd</url>
         </entry_point>
     </entry_points>
+    <environment_variables>
+        <environment_variable name="GALAXY_URL">${__app__.config.galaxy_infrastructure_url}</environment_variable>
+        <environment_variable name="API_KEY" strip="True">
+            #if $__user__:
+                #for $api_key in $__user__.api_keys:
+                    ${api_key.key}
+                    #break
+                #end for
+            #end if
+        </environment_variable>
+    </environment_variables>
     <command><![CDATA[
+        #import re
         ## ToDo: the key could be generated randomly
         export ASKO_load_url='http://localhost:6543' &&
         export ASKOMICS_API_KEY='abcd' &&
@@ -17,10 +29,18 @@
         export VIRT_Parameters_NumberOfBuffers='10000'  &&
         export VIRT_Parameters_MaxDirtyBuffers='6000'  &&
 
+        #set link_name = re.sub('[^\w_]', '_', $infile.element_identifier)
+        #if $infile.ext == 'tabular':
+            #set link_name = $link_name + '.tsv'
+        #elif $infile.ext == 'interval':
+            #set link_name = $link_name + '.bed'
+        #else:
+            #set link_name = $link_name + '.' + $infile.ext
+        #end if
+
         mkdir -p /import &&
-        cp '$infile' /import/foo.bed &&
+        ln -s '$infile' '/import/$link_name' &&
 
-        sed -i.bak 's@^/monitor_traffic.*@@g' /start.sh &&
         bash /start.sh
     ]]>
     </command>

From 0e5892c5ca71a73082405a6f8c7e5d92399405f3 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 13 Aug 2019 14:45:39 -0400
Subject: [PATCH 096/114] Update galaxy.yml.realtime example with
 outputs_to_working_directory: true

---
 config/galaxy.yml.realtime | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/config/galaxy.yml.realtime b/config/galaxy.yml.realtime
index 04ec6126aa9..09c7fb73632 100644
--- a/config/galaxy.yml.realtime
+++ b/config/galaxy.yml.realtime
@@ -25,4 +25,6 @@ uwsgi:
 
 galaxy:
 
+  outputs_to_working_directory: true
   realtime_prefix: realtime
+  realtime_map: database/realtime_map.sqlite

From 24c7c763794d7caabe9e44c86ad63ca4c3eb117d Mon Sep 17 00:00:00 2001
From: Bjoern Gruening <bjoern.gruening@gmail.com>
Date: Wed, 14 Aug 2019 09:15:09 +0200
Subject: [PATCH 097/114] remove complexity from ipython wrapper, not needed
 for this version

---
 .../interactive_tool_juypter_notebook.xml     | 73 ++++++++++---------
 1 file changed, 37 insertions(+), 36 deletions(-)

diff --git a/test/functional/tools/interactive_tool_juypter_notebook.xml b/test/functional/tools/interactive_tool_juypter_notebook.xml
index 833c6c8264a..0891cb4c5ed 100644
--- a/test/functional/tools/interactive_tool_juypter_notebook.xml
+++ b/test/functional/tools/interactive_tool_juypter_notebook.xml
@@ -1,6 +1,6 @@
 <tool id="interactive_tool_jupyter_notebook" tool_type="interactive" name="Interactive Jupyter Notebook" version="0.1">
     <requirements>
-        <container type="docker">bgruening/docker-jupyter-notebook:ie2</container>
+        <container type="docker">quay.io/bgruening/docker-jupyter-notebook:ie2</container>
     </requirements>
     <entry_points>
         <entry_point name="Jupyter Interactive Tool">
@@ -9,14 +9,9 @@
         </entry_point>
     </entry_points>
     <environment_variables>
-        <environment_variable name="HISTORY_ID" strip="True">
-            #if $jupyter_notebook:
-                ${__app__.security.encode_id($jupyter_notebook.history_id)}
-            #else:
-                ${__app__.security.encode_id($out_file1.history_id)}
-            #end if
-        </environment_variable>
-        <environment_variable name="REMOTE_HOST">TODO</environment_variable>
+        <environment_variable name="HISTORY_ID" strip="True">${__app__.security.encode_id($jupyter_notebook.history_id)}</environment_variable>
+        <environment_variable name="REMOTE_HOST">${__app__.config.galaxy_infrastructure_url}</environment_variable>
+        <environment_variable name="GALAXY_WEB_PORT">8080</environment_variable>
         <environment_variable name="GALAXY_URL">${__app__.config.galaxy_infrastructure_url}</environment_variable>
         <environment_variable name="API_KEY" strip="True">
             #if $__user__:
@@ -33,67 +28,73 @@
         mkdir -p ./jupyter/outputs/ &&
         mkdir -p ./jupyter/data &&
 
-        #for $i, $input in enumerate($inputs):
-            #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier))
-            ln -sf '$input' './jupyter/data/${cleaned_name}_${i}' &&
-        #end for
+        #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier))
+        ln -sf '$input' './jupyter/data/${cleaned_name}' &&
 
         ## change into the directory where the notebooks are located
         cd ./jupyter/ &&
+        export PATH=/home/jovyan/.local/bin:\$PATH &&
 
         #if $mode.mode_select == 'scratch':
             ## copy default notebook
             cp '$__tool_directory__/default_notebook.ipynb' ./ipython_galaxy_notebook.ipynb &&
-            /startup.sh
+            jupyter trust ./ipython_galaxy_notebook.ipynb &&
+            jupyter lab --no-browser --NotebookApp.shutdown_button=True &&
+            cp ./ipython_galaxy_notebook.ipynb '$jupyter_notebook'
+
         #else:
+            #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier))
+            cp '$mode.ipynb' ./${cleaned_name}.ipynb &&
+            jupyter trust ./${cleaned_name}.ipynb &&
 
-            #for $i, $ipynb in enumerate($mode.ipynbs):
-                #set $cleaned_name = re.sub('[^\w\-\.]', '_', str($input.element_identifier))
-                cp '$ipynb' ./${cleaned_name}_${i}.ipynb &&
-            #end for
             #if $mode.run_it:
                 jupyter nbconvert --to notebook --execute --output ./ipython_galaxy_notebook.ipynb --allow-errors  ./*.ipynb &&
-                mv ./ipython_galaxy_notebook.ipynb $jupyter_notebook
             #else:
-                /startup.sh
+                jupyter lab --no-browser --NotebookApp.shutdown_button=True &&
             #end if
-
+            cp ./ipython_galaxy_notebook.ipynb '$jupyter_notebook'
         #end if
     ]]>
     </command>
     <inputs>
 
         <conditional name="mode">
-            <param argument="mode_select" type="select" label="Do you already have a notebook?" help="If not, no problem we will provide you with a default one.">
+            <param name="mode_select" type="select" label="Do you already have a notebook?" help="If not, no problem we will provide you with a default one.">
                 <option value="scratch">Start with a fresh notebook</option>
                 <option value="previous">Load a previous notebook</option>
             </param>
             <when value="scratch"/>
             <when value="previous">
-                <param name="ipynbs" type="data" format="ipynb" multiple="true" label="IPython Notebook"/>
+                <param name="ipynb" type="data" format="ipynb" label="IPython Notebook"/>
                 <param name="run_it" type="boolean" truevalue="true" falsevalue="false" label="Execute notebook and return a new one."
                     help="This option is useful in workflows when you just want to execute a notebook and not dive into the webfrontend."/>
             </when>
         </conditional>
-
-        <param name="inputs" type="data" optional="true" multiple="true" label="Include data into the environment"/>
+        <param name="input" type="data" optional="true" label="Include data into the environment"/>
 
     </inputs>
     <outputs>
-        <data name="out_file1" format="txt">
-            <filter>(mode['mode_select'] == 'previous' and not mode['run_it']) or mode['mode_select'] == 'scratch'</filter>
-        </data>
-        <data name="jupyter_notebook" format="ipynb" label="Executed Notebook">
-            <filter>mode['mode_select'] == 'previous'</filter>
-            <filter>mode['run_it']</filter>
-        </data>
-        <!--data auto_format="true" name="collected_container_data">
-            <discover_datasets pattern="__designation__" directory="./jupyter/outputs/" visible="true" recurse="true" />
-        </data-->
+        <data name="jupyter_notebook" format="ipynb" label="Executed Notebook"></data>
     </outputs>
     <tests>
+        <test expect_num_outputs="1">
+            <param name="mode" value="previous" />
+            <param name="ipynb" value="test.ipynb" />
+            <param name="run_it" value="true" />
+            <output name="jupyter_notebook" file="test.ipynb" ftype="ipynb"/>
+        </test>
     </tests>
     <help>
-    Interactive tool.
+    The Jupyter Notebook is an open-source web application that allows you to create and share documents that contain live code, equations,
+    visualizations and narrative text. Uses include: data cleaning and transformation, numerical simulation, statistical modeling, data visualization,
+    machine learning, and much more.
+    
+    Galaxy offers you to use Jupyter Notebooks directly in Galaxy accessing and interacting with Galaxy datasets as you like. A very common use-case is to
+    do the heavy lifting and data reduction steps in Galaxy and the plotting and more `interactive` part on smaller datasets in Jupyter.
+    
+    You can start with a new Jupyter notebook from scratch or load an already existing one, e.g. from your collegue and execute it on your dataset.
+    If you have a defined input dataset you can even execute a Jupyter notebook in a workflow, given that the notebook is writing the output back to the history.
+    
+    You can import data into the notebook via a predefined `get()` function and write results back to Galaxy with a `put()` function.
     </help>
 </tool>

From 2fd4c13aba0eb38ed34ff775b30aeb8c403acaf0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= <bjoern@gruenings.eu>
Date: Wed, 14 Aug 2019 09:32:05 +0200
Subject: [PATCH 098/114] enhance documentation for developers

---
 config/job_conf.xml.realtime | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/config/job_conf.xml.realtime b/config/job_conf.xml.realtime
index a42e3305ad8..0d9b8901bc3 100644
--- a/config/job_conf.xml.realtime
+++ b/config/job_conf.xml.realtime
@@ -9,12 +9,21 @@
 
         <destination id="docker_local" runner="local">
           <param id="docker_enabled">true</param>
+          <!-- If you have not set 'outputs_to_working_directory: true' in galaxy.yml you can remove the docker_volumes setting. -->
           <param id="docker_volumes">$galaxy_root:ro,$tool_directory:ro,$job_directory:rw,$working_directory:rw,$default_file_path:ro</param>
           <param id="docker_sudo">false</param>
           <param id="docker_net">bridge</param>
           <param id="docker_auto_rm">true</param>
           <param id="require_container">true</param>
           <param id="docker_set_user"></param>
+          <!-- Interactive tools do need real hostnames or URLs to work - simply specifying IPs will not work.
+               If you develop interactive tools on your 'localhost' and don't have a proper domain name
+               you need to tell all Docker containers a hostname where Galaxy is running. 
+               This can be done via the --add-host parameter during the `docker run` command.
+               'bag' is here an arbritrary hostname that matches the IP address of your
+               Galaxy host. Make sure this hostname ('bag') is also set in your galaxy.yml file, e.g.
+               `galaxy_infrastructure_url: http://bag:8080`.
+          <!--param id="docker_run_extra_arguments">--add-host bag:132.230.153.81</param-->
         </destination>
 
         <destination id="docker_dispatch" runner="dynamic">

From c34fce8a5e4022d70ac563ba88f890956c504fbb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= <bjoern@gruenings.eu>
Date: Wed, 14 Aug 2019 09:36:27 +0200
Subject: [PATCH 099/114] add some more comments

---
 config/galaxy.yml.realtime | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/config/galaxy.yml.realtime b/config/galaxy.yml.realtime
index 09c7fb73632..6afbc7adf13 100644
--- a/config/galaxy.yml.realtime
+++ b/config/galaxy.yml.realtime
@@ -24,7 +24,12 @@ uwsgi:
 
 
 galaxy:
-
+  # outputs_to_working_directory will provide you with a better level of isolation. It is highly recommended to set
+  # this parameter with interactive tools.
   outputs_to_working_directory: true
   realtime_prefix: realtime
   realtime_map: database/realtime_map.sqlite
+  # If you develop interactive tools locally and do not have a full FQDN you can
+  # use an arbritrary one, e.g. 'bag' here, iff you set this hostname in your
+  # job_conf.xml as well (see the corresponding comment).
+  galaxy_infrastructure_url: http://bag:8080

From 0ba4ca5175211d5ebbc7ed99df146034f7ba064d Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Wed, 14 Aug 2019 10:08:40 -0400
Subject: [PATCH 100/114] cleanup comments

---
 config/galaxy.yml.realtime   |  8 ++++----
 config/job_conf.xml.realtime | 10 +++++-----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/config/galaxy.yml.realtime b/config/galaxy.yml.realtime
index 6afbc7adf13..2273cd80e05 100644
--- a/config/galaxy.yml.realtime
+++ b/config/galaxy.yml.realtime
@@ -25,11 +25,11 @@ uwsgi:
 
 galaxy:
   # outputs_to_working_directory will provide you with a better level of isolation. It is highly recommended to set
-  # this parameter with interactive tools.
+  # this parameter with InteractiveTools.
   outputs_to_working_directory: true
   realtime_prefix: realtime
   realtime_map: database/realtime_map.sqlite
-  # If you develop interactive tools locally and do not have a full FQDN you can
-  # use an arbritrary one, e.g. 'bag' here, iff you set this hostname in your
+  # If you develop InteractiveTools locally and do not have a full FQDN you can
+  # use an arbritrary one, e.g. 'my-hostname' here, if you set this hostname in your
   # job_conf.xml as well (see the corresponding comment).
-  galaxy_infrastructure_url: http://bag:8080
+  # galaxy_infrastructure_url: http://my-hostname:8080
diff --git a/config/job_conf.xml.realtime b/config/job_conf.xml.realtime
index 0d9b8901bc3..519cb95f109 100644
--- a/config/job_conf.xml.realtime
+++ b/config/job_conf.xml.realtime
@@ -16,14 +16,14 @@
           <param id="docker_auto_rm">true</param>
           <param id="require_container">true</param>
           <param id="docker_set_user"></param>
-          <!-- Interactive tools do need real hostnames or URLs to work - simply specifying IPs will not work.
+          <!-- InteractiveTools do need real hostnames or URLs to work - simply specifying IPs will not work.
                If you develop interactive tools on your 'localhost' and don't have a proper domain name
                you need to tell all Docker containers a hostname where Galaxy is running. 
                This can be done via the --add-host parameter during the `docker run` command.
-               'bag' is here an arbritrary hostname that matches the IP address of your
-               Galaxy host. Make sure this hostname ('bag') is also set in your galaxy.yml file, e.g.
-               `galaxy_infrastructure_url: http://bag:8080`.
-          <!--param id="docker_run_extra_arguments">--add-host bag:132.230.153.81</param-->
+               'my-hostname' here is an arbritrary hostname that matches the IP address of your
+               Galaxy host. Make sure this hostname ('my-hostname') is also set in your galaxy.yml file, e.g.
+               `galaxy_infrastructure_url: http://my-hostname:8080`. -->
+          <!--param id="docker_run_extra_arguments">--add-host my-hostname:10.1.2.3</param-->
         </destination>
 
         <destination id="docker_dispatch" runner="dynamic">

From 6953bfae06d48aca8c32a74ded963bb6d190c438 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Wed, 14 Aug 2019 10:09:23 -0400
Subject: [PATCH 101/114] update migration script version

---
 .../{0155_add_realtimetools.py => 0156_add_realtimetools.py}      | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename lib/galaxy/model/migrate/versions/{0155_add_realtimetools.py => 0156_add_realtimetools.py} (100%)

diff --git a/lib/galaxy/model/migrate/versions/0155_add_realtimetools.py b/lib/galaxy/model/migrate/versions/0156_add_realtimetools.py
similarity index 100%
rename from lib/galaxy/model/migrate/versions/0155_add_realtimetools.py
rename to lib/galaxy/model/migrate/versions/0156_add_realtimetools.py

From 4bda4c92bdb51a3aa1dc51336f34b8e0ff6cdbf9 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 20 Aug 2019 14:57:45 -0400
Subject: [PATCH 102/114] Make InteractiveTools enable-able.

---
 client/galaxy/scripts/layout/menu.js              | 13 ++++++++-----
 config/galaxy.yml.realtime                        |  1 +
 doc/source/admin/galaxy_options.rst               | 10 ++++++++++
 lib/galaxy/config/__init__.py                     |  1 +
 lib/galaxy/config/sample/galaxy.yml.sample        |  3 +++
 lib/galaxy/managers/configuration.py              |  1 +
 lib/galaxy/tools/__init__.py                      |  4 ++++
 lib/galaxy/webapps/galaxy/config_schema.yml       |  7 +++++++
 lib/galaxy/webapps/galaxy/controllers/realtime.py |  2 ++
 9 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/client/galaxy/scripts/layout/menu.js b/client/galaxy/scripts/layout/menu.js
index 10cb4da0ce2..84a60e03c3b 100644
--- a/client/galaxy/scripts/layout/menu.js
+++ b/client/galaxy/scripts/layout/menu.js
@@ -299,11 +299,6 @@ const Collection = Backbone.Collection.extend({
                         title: _l("Pages"),
                         url: "pages/list",
                         target: "__use_router__"
-                    },
-                    {
-                        title: _l("Active InteractiveTools"),
-                        url: "realtime_entry_points/list",
-                        target: "__use_router__"
                     }
                 ]
             };
@@ -314,6 +309,14 @@ const Collection = Backbone.Collection.extend({
                     target: "__use_router__"
                 });
             }
+            if (Galaxy.config.interactivetools_enable) {
+                userTab.menu[userTab.menu.length - 1].divider = true;
+                userTab.menu.push({
+                    title: _l("Active InteractiveTools"),
+                    url: "realtime_entry_points/list",
+                    target: "__use_router__"
+                });
+            }
         }
         this.add(userTab);
         return new $.Deferred().resolve().promise();
diff --git a/config/galaxy.yml.realtime b/config/galaxy.yml.realtime
index 2273cd80e05..5b0bea4e396 100644
--- a/config/galaxy.yml.realtime
+++ b/config/galaxy.yml.realtime
@@ -24,6 +24,7 @@ uwsgi:
 
 
 galaxy:
+  interactivetools_enable: true
   # outputs_to_working_directory will provide you with a better level of isolation. It is highly recommended to set
   # this parameter with InteractiveTools.
   outputs_to_working_directory: true
diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst
index c8ca18810e2..56a3ad5ec2d 100644
--- a/doc/source/admin/galaxy_options.rst
+++ b/doc/source/admin/galaxy_options.rst
@@ -1236,6 +1236,16 @@
 :Type: bool
 
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``interactivetools_enable``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Set this to true to enable InteractiveTools.
+:Default: ``false``
+:Type: bool
+
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 ``visualizations_visible``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py
index 1ae11fbe008..dcf693a4fff 100644
--- a/lib/galaxy/config/__init__.py
+++ b/lib/galaxy/config/__init__.py
@@ -737,6 +737,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration):
         self.dynamic_proxy_golang_api_key = kwargs.get("dynamic_proxy_golang_api_key", None)
 
         self.realtime_prefix = kwargs.get("realtime_prefix", "realtime")
+        self.interactivetools_enable = string_as_bool(kwargs.get('interactivetools_enable', False))
 
         # Default chunk size for chunkable datatypes -- 64k
         self.display_chunk_size = int(kwargs.get('display_chunk_size', 65536))
diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample
index 52baec7990c..f9db07f32c6 100644
--- a/lib/galaxy/config/sample/galaxy.yml.sample
+++ b/lib/galaxy/config/sample/galaxy.yml.sample
@@ -423,6 +423,9 @@ galaxy:
   # scenarios than the watchdog default.
   #watch_tool_data_dir: 'false'
 
+  # Enable InteractiveTools by setting to true
+  #interactivetools_enable: false
+
   # File containing old-style genome builds
   #builds_file_path: tool-data/shared/ucsc/builds.txt
 
diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py
index 32c2464ddca..02f4bafd219 100644
--- a/lib/galaxy/managers/configuration.py
+++ b/lib/galaxy/managers/configuration.py
@@ -79,6 +79,7 @@ class ConfigSerializer(base.ModelSerializer):
             'require_login'                     : _defaults_to(None),
             'inactivity_box_content'            : _defaults_to(None),
             'visualizations_visible'            : _defaults_to(True),
+            'interactivetools_enable'           : _defaults_to(False),
             'message_box_content'               : _defaults_to(None),
             'message_box_visible'               : _defaults_to(False),
             'message_box_class'                 : _defaults_to('info'),
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index eb2be32404b..715c0ccf8e2 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -2449,6 +2449,10 @@ class RealTimeTool(Tool):
     tool_type = 'interactive'
     produces_entry_points = True
 
+    def __init__(self, config_file, tool_source, app, **kwd):
+        assert app.config.interactivetools_enable, ValueError('Trying to load an InteractiveTool, but InteractiveTools are not enabled.')
+        super(RealTimeTool, self).__init__(config_file, tool_source, app, **kwd)
+
     def __remove_realtime_by_job(self, job):
         if job:
             eps = job.realtimetool_entry_points
diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml
index 85c1c617386..2bb3dfd771e 100644
--- a/lib/galaxy/webapps/galaxy/config_schema.yml
+++ b/lib/galaxy/webapps/galaxy/config_schema.yml
@@ -928,6 +928,13 @@ mapping:
           Galaxy server, but contains a redirect to a third-party server, tricking a
           Galaxy user to access said site.
 
+      interactivetools_enable:
+        type: bool
+        default: false
+        required: false
+        desc: |
+          Enable InteractiveTools.
+
       visualizations_visible:
         type: bool
         default: true
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index e43a92d1e60..785b18aec57 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -63,6 +63,8 @@ class RealTime(BaseUIController):
     @web.expose_api_anonymous
     def list(self, trans, **kwargs):
         """List all available realtimetools"""
+        if not trans.app.config.interactivetools_enable:
+            raise web.httpexceptions.HTTPNotFound()
         operation = kwargs.get('operation', None)
         message = None
         status = None

From ab6e934fd3dd7a3ce2467b093740cd9ba308f0a9 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 20 Aug 2019 15:58:13 -0400
Subject: [PATCH 103/114] minor typo

---
 lib/galaxy/model/migrate/versions/0156_add_realtimetools.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/model/migrate/versions/0156_add_realtimetools.py b/lib/galaxy/model/migrate/versions/0156_add_realtimetools.py
index c60b2b5defc..3da95384133 100644
--- a/lib/galaxy/model/migrate/versions/0156_add_realtimetools.py
+++ b/lib/galaxy/model/migrate/versions/0156_add_realtimetools.py
@@ -1,5 +1,5 @@
 """
-Migration script to add a new tables for RealTimeTools.
+Migration script to add new tables for RealTimeTools.
 """
 from __future__ import print_function
 

From 00e575fa1d1c203eee6c5a0c7559db993500f79b Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 20 Aug 2019 16:13:30 -0400
Subject: [PATCH 104/114] interactive_tool_juypter_notebook.xml ->
 interactive_tool_jupyter_notebook.xml

---
 ...juypter_notebook.xml => interactive_tool_jupyter_notebook.xml} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename test/functional/tools/{interactive_tool_juypter_notebook.xml => interactive_tool_jupyter_notebook.xml} (100%)

diff --git a/test/functional/tools/interactive_tool_juypter_notebook.xml b/test/functional/tools/interactive_tool_jupyter_notebook.xml
similarity index 100%
rename from test/functional/tools/interactive_tool_juypter_notebook.xml
rename to test/functional/tools/interactive_tool_jupyter_notebook.xml

From 41d7155fb6defa3ec83fdaaa93182b9935b6f53d Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 20 Aug 2019 16:20:48 -0400
Subject: [PATCH 105/114] Add requires_domain to entry_point. Default is False.
 Currently only True works.

---
 lib/galaxy/tool_util/parser/xml.py | 3 ++-
 lib/galaxy/tools/__init__.py       | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py
index 3e06ebe430b..c3c05d38be2 100644
--- a/lib/galaxy/tool_util/parser/xml.py
+++ b/lib/galaxy/tool_util/parser/xml.py
@@ -216,7 +216,8 @@ class XmlToolSource(ToolSource):
             name = ep_el.get('name', None)
             if name:
                 name = name.strip()
-            rtt.append(dict(port=port, url=url, name=name))
+            requires_domain = string_as_bool(ep_el.attrib.get("requires_domain", False))
+            rtt.append(dict(port=port, url=url, name=name, requires_domain=requires_domain))
         return rtt
 
     def parse_hidden(self):
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 715c0ccf8e2..60325c5b003 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -2452,6 +2452,8 @@ class RealTimeTool(Tool):
     def __init__(self, config_file, tool_source, app, **kwd):
         assert app.config.interactivetools_enable, ValueError('Trying to load an InteractiveTool, but InteractiveTools are not enabled.')
         super(RealTimeTool, self).__init__(config_file, tool_source, app, **kwd)
+        for port in self.ports:
+            assert port.get('requires_domain', None), ValueError('InteractiveTools currently only work when requires_domain is True for each entry_point.')
 
     def __remove_realtime_by_job(self, job):
         if job:

From f16da013add2d7ba8ce1c9455402bac9c7124d56 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 20 Aug 2019 16:30:41 -0400
Subject: [PATCH 106/114] Set requires_domain="True" for all examples.

---
 test/functional/tools/interactive_tool_askomics.xml         | 2 +-
 test/functional/tools/interactive_tool_bam_iobio.xml        | 2 +-
 test/functional/tools/interactive_tool_cellxgene.xml        | 2 +-
 test/functional/tools/interactive_tool_ethercalc.xml        | 2 +-
 test/functional/tools/interactive_tool_hicbrowser.xml       | 2 +-
 test/functional/tools/interactive_tool_jupyter_notebook.xml | 2 +-
 test/functional/tools/interactive_tool_neo4j.xml            | 2 +-
 test/functional/tools/interactive_tool_phinch.xml           | 2 +-
 test/functional/tools/interactive_tool_simple.xml           | 4 ++--
 test/functional/tools/interactive_tool_two_entry_points.xml | 6 +++---
 test/functional/tools/realtimetool_ethercalc.xml            | 2 +-
 11 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/test/functional/tools/interactive_tool_askomics.xml b/test/functional/tools/interactive_tool_askomics.xml
index 3014321a164..de9827b3eb5 100644
--- a/test/functional/tools/interactive_tool_askomics.xml
+++ b/test/functional/tools/interactive_tool_askomics.xml
@@ -4,7 +4,7 @@
         <container type="docker">quay.io/askomics/askomics-ie:17.12_g19.09</container>
     </requirements>
     <entry_points>
-        <entry_point name="AskOmics instance on $infile.display_name">
+        <entry_point name="AskOmics instance on $infile.display_name" requires_domain="True">
             <port>6543</port>
             <url>/login_api_gie?key=abcd</url>
         </entry_point>
diff --git a/test/functional/tools/interactive_tool_bam_iobio.xml b/test/functional/tools/interactive_tool_bam_iobio.xml
index a2006e5a5ee..2f1c531861c 100644
--- a/test/functional/tools/interactive_tool_bam_iobio.xml
+++ b/test/functional/tools/interactive_tool_bam_iobio.xml
@@ -3,7 +3,7 @@
         <container type="docker">qiaoy/iobio-bundle.bam-iobio:1.0-ondemand</container>
     </requirements>
     <entry_points>
-        <entry_point name="BAM io.bio visualisation of $infile.display_name">
+        <entry_point name="BAM io.bio visualisation of $infile.display_name" requires_domain="True">
             <port>80</port>
             <url><![CDATA[/?bam=http://localhost/tmp/bamfile.bam&region=1]]></url>
         </entry_point>
diff --git a/test/functional/tools/interactive_tool_cellxgene.xml b/test/functional/tools/interactive_tool_cellxgene.xml
index 0eb659681e5..546bd7ab58f 100644
--- a/test/functional/tools/interactive_tool_cellxgene.xml
+++ b/test/functional/tools/interactive_tool_cellxgene.xml
@@ -3,7 +3,7 @@
         <container type="docker">quay.io/galaxy/cellxgene-galaxy-ie:ie2</container>
     </requirements>
     <entry_points>
-        <entry_point name="Cellxgene Single Cell Visualisation on $infile.display_name">
+        <entry_point name="Cellxgene Single Cell Visualisation on $infile.display_name" requires_domain="True">
             <port>80</port>
         </entry_point>
     </entry_points>
diff --git a/test/functional/tools/interactive_tool_ethercalc.xml b/test/functional/tools/interactive_tool_ethercalc.xml
index db37831f6cd..88676c1e997 100644
--- a/test/functional/tools/interactive_tool_ethercalc.xml
+++ b/test/functional/tools/interactive_tool_ethercalc.xml
@@ -3,7 +3,7 @@
         <container type="docker">shiltemann/ethercalc-galaxy-ie:17.05</container>
     </requirements>
     <entry_points>
-        <entry_point name="EtherCalc instance on $infile.display_name">
+        <entry_point name="EtherCalc instance on $infile.display_name" requires_domain="True">
             <port>8000</port>
         </entry_point>
     </entry_points>
diff --git a/test/functional/tools/interactive_tool_hicbrowser.xml b/test/functional/tools/interactive_tool_hicbrowser.xml
index 3d50401ec5f..7aaa8a3e8e7 100644
--- a/test/functional/tools/interactive_tool_hicbrowser.xml
+++ b/test/functional/tools/interactive_tool_hicbrowser.xml
@@ -3,7 +3,7 @@
         <container type="docker">bgruening/hicbrowser</container>
     </requirements>
     <entry_points>
-        <entry_point name="HiCBrowser on $infile.display_name">
+        <entry_point name="HiCBrowser on $infile.display_name" requires_domain="True">
             <port>80</port>
         </entry_point>
     </entry_points>
diff --git a/test/functional/tools/interactive_tool_jupyter_notebook.xml b/test/functional/tools/interactive_tool_jupyter_notebook.xml
index 0891cb4c5ed..336a9a00b5a 100644
--- a/test/functional/tools/interactive_tool_jupyter_notebook.xml
+++ b/test/functional/tools/interactive_tool_jupyter_notebook.xml
@@ -3,7 +3,7 @@
         <container type="docker">quay.io/bgruening/docker-jupyter-notebook:ie2</container>
     </requirements>
     <entry_points>
-        <entry_point name="Jupyter Interactive Tool">
+        <entry_point name="Jupyter Interactive Tool" requires_domain="True">
             <port>8888</port>
             <url>ipython/tree</url>
         </entry_point>
diff --git a/test/functional/tools/interactive_tool_neo4j.xml b/test/functional/tools/interactive_tool_neo4j.xml
index d1e21dec82a..0b31a8b303e 100644
--- a/test/functional/tools/interactive_tool_neo4j.xml
+++ b/test/functional/tools/interactive_tool_neo4j.xml
@@ -3,7 +3,7 @@
         <container type="docker">quay.io/sanbi-sa/neo_ie:3.1.9</container>
     </requirements>
     <entry_points>
-        <entry_point name="Neo4j on $infile.display_name">
+        <entry_point name="Neo4j on $infile.display_name" requires_domain="True">
             <port>80</port>
         </entry_point>
         <!-- I get some web-socket error that 7687 is not reachable. 
diff --git a/test/functional/tools/interactive_tool_phinch.xml b/test/functional/tools/interactive_tool_phinch.xml
index e7c6706be5f..d77c9274ad3 100644
--- a/test/functional/tools/interactive_tool_phinch.xml
+++ b/test/functional/tools/interactive_tool_phinch.xml
@@ -3,7 +3,7 @@
         <container type="docker">shiltemann/docker-phinch-galaxy:16.04</container>
     </requirements>
     <entry_points>
-        <entry_point name="Phinch Visualisation of $infile.display_name">
+        <entry_point name="Phinch Visualisation of $infile.display_name" requires_domain="True">
             <port>80</port>
         </entry_point>
     </entry_points>
diff --git a/test/functional/tools/interactive_tool_simple.xml b/test/functional/tools/interactive_tool_simple.xml
index 9c6c48b9059..766af72dce1 100644
--- a/test/functional/tools/interactive_tool_simple.xml
+++ b/test/functional/tools/interactive_tool_simple.xml
@@ -1,9 +1,9 @@
-<tool id="realtimetool_simple" name="realtimetool_simple" tool_type="interactive" version="0.1">
+<tool id="interactive_tool_simple" name="realtimetool_simple" tool_type="interactive" version="0.1">
     <requirements>
         <container type="docker">galaxy/test-http-example:0.1</container>
     </requirements>
     <entry_points>
-        <entry_point name="Simple">
+        <entry_point name="Simple" requires_domain="True">
             <port>7000</port>
             <url>/</url>
         </entry_point>
diff --git a/test/functional/tools/interactive_tool_two_entry_points.xml b/test/functional/tools/interactive_tool_two_entry_points.xml
index dc45d9caa86..1af1efe6f65 100644
--- a/test/functional/tools/interactive_tool_two_entry_points.xml
+++ b/test/functional/tools/interactive_tool_two_entry_points.xml
@@ -1,13 +1,13 @@
-<tool id="realtimetool_two_entry_points" name="realtimetool_two_entry_points" tool_type="interactive" version="0.1">
+<tool id="interactive_tool_two_entry_points" name="realtimetool_two_entry_points" tool_type="interactive" version="0.1">
     <requirements>
         <container type="docker">galaxy/test-http-example:0.1</container>
     </requirements>
     <entry_points>
-        <entry_point name="Server 1">
+        <entry_point name="Server 1" requires_domain="True">
             <port>7000</port>
             <url>/</url>
         </entry_point>
-        <entry_point name="Server 2">
+        <entry_point name="Server 2" requires_domain="True">
             <port>7001</port>
             <url>/</url>
         </entry_point>
diff --git a/test/functional/tools/realtimetool_ethercalc.xml b/test/functional/tools/realtimetool_ethercalc.xml
index 387e931e405..9e592534126 100644
--- a/test/functional/tools/realtimetool_ethercalc.xml
+++ b/test/functional/tools/realtimetool_ethercalc.xml
@@ -3,7 +3,7 @@
         <container type="docker">shiltemann/ethercalc-galaxy-ie:17.05</container>
     </requirements>
     <entry_points>
-        <entry_point name="ethercalc">
+        <entry_point name="ethercalc" requires_domain="True">
             <port>8000</port>
         </entry_point>
     </entry_points>

From 725016230ef56f1e9f25d3018f3643dbfcc539cf Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 20 Aug 2019 16:36:29 -0400
Subject: [PATCH 107/114] Add warning on backwards compatibility of __app__ in
 future.

---
 test/functional/tools/interactive_tool_askomics.xml         | 2 +-
 test/functional/tools/interactive_tool_jupyter_notebook.xml | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/functional/tools/interactive_tool_askomics.xml b/test/functional/tools/interactive_tool_askomics.xml
index de9827b3eb5..71c3fe9a437 100644
--- a/test/functional/tools/interactive_tool_askomics.xml
+++ b/test/functional/tools/interactive_tool_askomics.xml
@@ -10,7 +10,7 @@
         </entry_point>
     </entry_points>
     <environment_variables>
-        <environment_variable name="GALAXY_URL">${__app__.config.galaxy_infrastructure_url}</environment_variable>
+        <environment_variable name="GALAXY_URL">${__app__.config.galaxy_infrastructure_url}</environment_variable> <!-- FIXME: Warning: The use of __app__ is deprecated and will break backward compatibility in the near future -->
         <environment_variable name="API_KEY" strip="True">
             #if $__user__:
                 #for $api_key in $__user__.api_keys:
diff --git a/test/functional/tools/interactive_tool_jupyter_notebook.xml b/test/functional/tools/interactive_tool_jupyter_notebook.xml
index 336a9a00b5a..27313f65527 100644
--- a/test/functional/tools/interactive_tool_jupyter_notebook.xml
+++ b/test/functional/tools/interactive_tool_jupyter_notebook.xml
@@ -9,10 +9,10 @@
         </entry_point>
     </entry_points>
     <environment_variables>
-        <environment_variable name="HISTORY_ID" strip="True">${__app__.security.encode_id($jupyter_notebook.history_id)}</environment_variable>
-        <environment_variable name="REMOTE_HOST">${__app__.config.galaxy_infrastructure_url}</environment_variable>
+        <environment_variable name="HISTORY_ID" strip="True">${__app__.security.encode_id($jupyter_notebook.history_id)}</environment_variable> <!-- FIXME: Warning: The use of __app__ is deprecated and will break backward compatibility in the near future -->
+        <environment_variable name="REMOTE_HOST">${__app__.config.galaxy_infrastructure_url}</environment_variable> <!-- FIXME: Warning: The use of __app__ is deprecated and will break backward compatibility in the near future -->
         <environment_variable name="GALAXY_WEB_PORT">8080</environment_variable>
-        <environment_variable name="GALAXY_URL">${__app__.config.galaxy_infrastructure_url}</environment_variable>
+        <environment_variable name="GALAXY_URL">${__app__.config.galaxy_infrastructure_url}</environment_variable> <!-- FIXME: Warning: The use of __app__ is deprecated and will break backward compatibility in the near future -->
         <environment_variable name="API_KEY" strip="True">
             #if $__user__:
                 #for $api_key in $__user__.api_keys:

From 34ec3bfd3a76a852dcbc9e637af395501e23a5ba Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 20 Aug 2019 16:38:20 -0400
Subject: [PATCH 108/114] FIXME note on user's API key in tool.

---
 test/functional/tools/interactive_tool_jupyter_notebook.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/functional/tools/interactive_tool_jupyter_notebook.xml b/test/functional/tools/interactive_tool_jupyter_notebook.xml
index 27313f65527..9dd3e36d3ee 100644
--- a/test/functional/tools/interactive_tool_jupyter_notebook.xml
+++ b/test/functional/tools/interactive_tool_jupyter_notebook.xml
@@ -20,7 +20,7 @@
                     #break
                 #end for
             #end if
-        </environment_variable>
+        </environment_variable> <!-- FIXME: We should have a better way to get user's API key -->
     </environment_variables>
     <command detect_errors="aggressive"><![CDATA[
         #import re

From a3167bd3b9e79c7aef93f84bde5ef499953d7565 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 20 Aug 2019 16:44:12 -0400
Subject: [PATCH 109/114] Update galaxy tool XSD.

---
 lib/galaxy/tool_util/xsd/galaxy.xsd | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index e12c94a134e..4e99440dc70 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -269,6 +269,11 @@ to provide access to graphical tools in real-time.
             <xs:documentation xml:lang="en">This value defines the name of the entry point.</xs:documentation>
           </xs:annotation>
         </xs:attribute>
+        <xs:attribute name="requires_domain" type="PermissiveBoolean" default="false">
+          <xs:annotation>
+            <xs:documentation xml:lang="en">This value declares if domain-based proxying is required. Default is False. Currently only works when True.</xs:documentation>
+          </xs:annotation>
+        </xs:attribute>
   </xs:complexType>
   <xs:complexType name="EntryPointPort" mixed="true">
     <xs:annotation>

From 55c929768cb4e2fd9f3169b5359eb570462dbf9b Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 20 Aug 2019 16:58:55 -0400
Subject: [PATCH 110/114] more realtime to interactivetools name changes

---
 .../{galaxy.yml.realtime => galaxy.yml.interactivetools}  | 8 ++++----
 lib/galaxy/config/__init__.py                             | 7 ++++---
 .../key_type_token_mapping.py                             | 2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)
 rename config/{galaxy.yml.realtime => galaxy.yml.interactivetools} (84%)
 rename scripts/{realtime => interactivetools}/key_type_token_mapping.py (98%)

diff --git a/config/galaxy.yml.realtime b/config/galaxy.yml.interactivetools
similarity index 84%
rename from config/galaxy.yml.realtime
rename to config/galaxy.yml.interactivetools
index 5b0bea4e396..6b196a42dbe 100644
--- a/config/galaxy.yml.realtime
+++ b/config/galaxy.yml.interactivetools
@@ -12,8 +12,8 @@ uwsgi:
 
   module: galaxy.webapps.galaxy.buildapp:uwsgi_app()
 
-  realtime_map: database/realtime_map.sqlite
-  python-raw: scripts/realtime/key_type_token_mapping.py
+  interactivetools_map: database/interactivetools_map.sqlite
+  python-raw: scripts/interactivetools/key_type_token_mapping.py
   route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.localhost:8080)$ goto:realtime
   route-run: goto:endendend
   route-label: realtime
@@ -28,8 +28,8 @@ galaxy:
   # outputs_to_working_directory will provide you with a better level of isolation. It is highly recommended to set
   # this parameter with InteractiveTools.
   outputs_to_working_directory: true
-  realtime_prefix: realtime
-  realtime_map: database/realtime_map.sqlite
+  interactivetools_prefix: realtime
+  interactivetools_map: database/interactivetools_map.sqlite
   # If you develop InteractiveTools locally and do not have a full FQDN you can
   # use an arbritrary one, e.g. 'my-hostname' here, if you set this hostname in your
   # job_conf.xml as well (see the corresponding comment).
diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py
index dcf693a4fff..e534d592f2a 100644
--- a/lib/galaxy/config/__init__.py
+++ b/lib/galaxy/config/__init__.py
@@ -708,8 +708,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration):
         self.fluent_log = string_as_bool(kwargs.get('fluent_log', False))
         self.fluent_host = kwargs.get('fluent_host', 'localhost')
         self.fluent_port = int(kwargs.get('fluent_port', 24224))
-        # RealTimeTools propagator mapping file
-        self.realtime_map = self.resolve_path(kwargs.get("realtime_map", "database/realtime_map.sqlite"))
+
         # directory where the visualization registry searches for plugins
         self.visualization_plugins_directory = kwargs.get(
             'visualization_plugins_directory', 'config/plugins/visualizations')
@@ -736,7 +735,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration):
         self.dynamic_proxy_golang_docker_address = kwargs.get("dynamic_proxy_golang_docker_address", "unix:///var/run/docker.sock")
         self.dynamic_proxy_golang_api_key = kwargs.get("dynamic_proxy_golang_api_key", None)
 
-        self.realtime_prefix = kwargs.get("realtime_prefix", "realtime")
+        # InteractiveTools propagator mapping file
+        self.realtime_map = self.resolve_path(kwargs.get("interactivetools_map", "database/interactivetools_map.sqlite"))
+        self.realtime_prefix = kwargs.get("interactivetools_prefix", "realtime")
         self.interactivetools_enable = string_as_bool(kwargs.get('interactivetools_enable', False))
 
         # Default chunk size for chunkable datatypes -- 64k
diff --git a/scripts/realtime/key_type_token_mapping.py b/scripts/interactivetools/key_type_token_mapping.py
similarity index 98%
rename from scripts/realtime/key_type_token_mapping.py
rename to scripts/interactivetools/key_type_token_mapping.py
index 3461c5c5783..fdbaa499e84 100644
--- a/scripts/realtime/key_type_token_mapping.py
+++ b/scripts/interactivetools/key_type_token_mapping.py
@@ -6,7 +6,7 @@ from time import time
 import uwsgi
 
 
-realtime_db_file = uwsgi.opt["realtime_map"]
+realtime_db_file = uwsgi.opt["interactivetools_map"]
 db_conn = sqlite3.connect(realtime_db_file)
 
 

From d7e99c7f5df6872432f8ab0617c7ed93d5260e40 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 20 Aug 2019 17:11:15 -0400
Subject: [PATCH 111/114] more realtimetool --> interactivetool

---
 lib/galaxy/model/mapping.py                        |  2 +-
 ...altimetools.py => 0156_add_interactivetools.py} | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)
 rename lib/galaxy/model/migrate/versions/{0156_add_realtimetools.py => 0156_add_interactivetools.py} (83%)

diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py
index 1b07bb74e1e..4215e04794a 100644
--- a/lib/galaxy/model/mapping.py
+++ b/lib/galaxy/model/mapping.py
@@ -793,7 +793,7 @@ model.GenomeIndexToolData.table = Table(
     Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True))
 
 model.RealTimeToolEntryPoint.table = Table(
-    "realtimetool_entry_point", metadata,
+    "interactivetool_entry_point", metadata,
     Column("id", Integer, primary_key=True),
     Column("job_id", Integer, ForeignKey("job.id"), index=True),
     Column("name", TEXT),
diff --git a/lib/galaxy/model/migrate/versions/0156_add_realtimetools.py b/lib/galaxy/model/migrate/versions/0156_add_interactivetools.py
similarity index 83%
rename from lib/galaxy/model/migrate/versions/0156_add_realtimetools.py
rename to lib/galaxy/model/migrate/versions/0156_add_interactivetools.py
index 3da95384133..f1c7687fc32 100644
--- a/lib/galaxy/model/migrate/versions/0156_add_realtimetools.py
+++ b/lib/galaxy/model/migrate/versions/0156_add_interactivetools.py
@@ -1,5 +1,5 @@
 """
-Migration script to add new tables for RealTimeTools.
+Migration script to add new tables for InteractiveTools.
 """
 from __future__ import print_function
 
@@ -13,8 +13,8 @@ from galaxy.model.orm.now import now
 log = logging.getLogger(__name__)
 metadata = MetaData()
 
-realtimetool_entry_point = Table(
-    "realtimetool_entry_point", metadata,
+interactivetool_entry_point = Table(
+    "interactivetool_entry_point", metadata,
     Column("id", Integer, primary_key=True),
     Column("job_id", Integer, ForeignKey("job.id"), index=True),
     Column("name", TEXT),
@@ -52,9 +52,9 @@ def upgrade(migrate_engine):
         log.exception("Failed to create job_container_association table")
 
     try:
-        realtimetool_entry_point.create()
+        interactivetool_entry_point.create()
     except Exception:
-        log.exception("Failed to create realtimetool_entry_point table")
+        log.exception("Failed to create interactivetool_entry_point table")
 
 
 def downgrade(migrate_engine):
@@ -67,6 +67,6 @@ def downgrade(migrate_engine):
         log.exception("Failed to drop job_container_association table")
 
     try:
-        realtimetool_entry_point.drop()
+        interactivetool_entry_point.drop()
     except Exception:
-        log.exception("Failed to drop realtimetool_entry_point table")
+        log.exception("Failed to drop interactivetool_entry_point table")

From 7732faec07c4e23a619711250b2c9d32c27e0f11 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 20 Aug 2019 17:17:11 -0400
Subject: [PATCH 112/114] more realtimetool --> interactivetool

---
 config/galaxy.yml.interactivetools | 8 ++++----
 lib/galaxy/config/__init__.py      | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/config/galaxy.yml.interactivetools b/config/galaxy.yml.interactivetools
index 6b196a42dbe..35baae11306 100644
--- a/config/galaxy.yml.interactivetools
+++ b/config/galaxy.yml.interactivetools
@@ -14,10 +14,10 @@ uwsgi:
 
   interactivetools_map: database/interactivetools_map.sqlite
   python-raw: scripts/interactivetools/key_type_token_mapping.py
-  route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.localhost:8080)$ goto:realtime
+  route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(interactivetool\.localhost:8080)$ goto:interactivetool
   route-run: goto:endendend
-  route-label: realtime
-  route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.localhost:8080)$ rpcvar:TARGET_HOST rtt_key_type_token_mapper_cached $2 $1 $3 $4 $0 5
+  route-label: interactivetool
+  route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(interactivetool\.localhost:8080)$ rpcvar:TARGET_HOST rtt_key_type_token_mapper_cached $2 $1 $3 $4 $0 5
   route-if-not: empty:${TARGET_HOST} httpdumb:${TARGET_HOST}
   route: .* break:404 Not Found
   route-label: endendend
@@ -28,7 +28,7 @@ galaxy:
   # outputs_to_working_directory will provide you with a better level of isolation. It is highly recommended to set
   # this parameter with InteractiveTools.
   outputs_to_working_directory: true
-  interactivetools_prefix: realtime
+  interactivetools_prefix: interactivetool
   interactivetools_map: database/interactivetools_map.sqlite
   # If you develop InteractiveTools locally and do not have a full FQDN you can
   # use an arbritrary one, e.g. 'my-hostname' here, if you set this hostname in your
diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py
index e534d592f2a..14546d08f75 100644
--- a/lib/galaxy/config/__init__.py
+++ b/lib/galaxy/config/__init__.py
@@ -737,7 +737,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration):
 
         # InteractiveTools propagator mapping file
         self.realtime_map = self.resolve_path(kwargs.get("interactivetools_map", "database/interactivetools_map.sqlite"))
-        self.realtime_prefix = kwargs.get("interactivetools_prefix", "realtime")
+        self.realtime_prefix = kwargs.get("interactivetools_prefix", "interactivetool")
         self.interactivetools_enable = string_as_bool(kwargs.get('interactivetools_enable', False))
 
         # Default chunk size for chunkable datatypes -- 64k

From 753d25899e0b39e7cce2fd369fd3fa8245b687b4 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Tue, 20 Aug 2019 17:26:12 -0400
Subject: [PATCH 113/114] more realtimetool --> interactivetool

---
 lib/galaxy/managers/realtime.py                   | 12 ++++++------
 lib/galaxy/model/__init__.py                      |  2 +-
 lib/galaxy/model/mapping.py                       |  4 ++--
 lib/galaxy/webapps/galaxy/controllers/realtime.py |  4 ++--
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/lib/galaxy/managers/realtime.py b/lib/galaxy/managers/realtime.py
index d050a3593f6..9b0de4c44d0 100644
--- a/lib/galaxy/managers/realtime.py
+++ b/lib/galaxy/managers/realtime.py
@@ -158,7 +158,7 @@ class RealTimeManager(object):
     def create_entry_points(self, job, tool, entry_points=None, flush=True):
         entry_points = entry_points or tool.ports
         for entry in entry_points:
-            ep = self.model.RealTimeToolEntryPoint(job=job, tool_port=entry['port'], entry_url=entry['url'], name=entry['name'])
+            ep = self.model.InteractiveToolEntryPoint(job=job, tool_port=entry['port'], entry_url=entry['url'], name=entry['name'])
             self.sa_session.add(ep)
         if flush:
             self.sa_session.flush()
@@ -173,7 +173,7 @@ class RealTimeManager(object):
         for ep in job.realtimetool_entry_points:
             port_dict = ports_dict.get(str(ep.tool_port), None)
             if port_dict is None:
-                log.error("Did not find port to assign to RealTimeToolEntryPoint by tool port: %s.", ep.tool_port)
+                log.error("Did not find port to assign to InteractiveToolEntryPoint by tool port: %s.", ep.tool_port)
                 not_configured.append(ep)
             else:
                 ep.host = port_dict['host']
@@ -217,7 +217,7 @@ class RealTimeManager(object):
                     query = query.filter(or_(*t))
             return query
         jobs = build_and_apply_filters(jobs, trans.app.model.Job.non_ready_states, lambda s: trans.app.model.Job.state == s)
-        return trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).filter(trans.app.model.RealTimeToolEntryPoint.job_id.in_([job.id for job in jobs]))
+        return trans.sa_session.query(trans.app.model.InteractiveToolEntryPoint).filter(trans.app.model.InteractiveToolEntryPoint.job_id.in_([job.id for job in jobs]))
 
     def can_access_job(self, trans, job):
         if job:
@@ -247,14 +247,14 @@ class RealTimeManager(object):
             self.remove_entry_point(entry_point)
             job = entry_point.job
             if not job.finished:
-                log.debug('Stopping Job: %s for RealTimeToolEntryPoint: %s', job, entry_point)
+                log.debug('Stopping Job: %s for InteractiveToolEntryPoint: %s', job, entry_point)
                 job.mark_deleted(trans.app.config.track_jobs_in_database)
                 # This self.job_manager.stop(job) does nothing without changing job.state, manually or e.g. with .mark_deleted()
                 self.job_manager.stop(job)
                 trans.sa_session.add(job)
                 trans.sa_session.flush()
         except Exception as e:
-            log.debug('Unable to stop job for RealTimeToolEntryPoint (%s): %s', entry_point, e)
+            log.debug('Unable to stop job for InteractiveToolEntryPoint (%s): %s', entry_point, e)
             return False
         return True
 
@@ -284,7 +284,7 @@ class RealTimeManager(object):
             return rval
 
     def access_entry_point_target(self, trans, entry_point_id):
-        entry_point = trans.sa_session.query(model.RealTimeToolEntryPoint).get(entry_point_id)
+        entry_point = trans.sa_session.query(model.InteractiveToolEntryPoint).get(entry_point_id)
         if self.app.realtime_manager.can_access_entry_point(trans, entry_point):
             if entry_point.active:
                 return self.target_if_active(entry_point)
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 0e2fbf0725e..19f5aeffa45 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -1490,7 +1490,7 @@ class JobContainerAssociation(RepresentById):
         self.container_info = container_info or {}
 
 
-class RealTimeToolEntryPoint(Dictifiable, RepresentById):
+class InteractiveToolEntryPoint(Dictifiable, RepresentById):
     dict_collection_visible_keys = ['id', 'name', 'active']
     dict_element_visible_keys = ['id', 'name', 'active']
 
diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py
index 4215e04794a..80ec3753df6 100644
--- a/lib/galaxy/model/mapping.py
+++ b/lib/galaxy/model/mapping.py
@@ -792,7 +792,7 @@ model.GenomeIndexToolData.table = Table(
     Column("indexer", String(64)),
     Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True))
 
-model.RealTimeToolEntryPoint.table = Table(
+model.InteractiveToolEntryPoint.table = Table(
     "interactivetool_entry_point", metadata,
     Column("id", Integer, primary_key=True),
     Column("job_id", Integer, ForeignKey("job.id"), index=True),
@@ -2263,7 +2263,7 @@ mapper(model.GenomeIndexToolData, model.GenomeIndexToolData.table, properties=di
     transfer=relation(model.TransferJob, backref='transfer_job')
 ))
 
-mapper(model.RealTimeToolEntryPoint, model.RealTimeToolEntryPoint.table, properties=dict(
+mapper(model.InteractiveToolEntryPoint, model.InteractiveToolEntryPoint.table, properties=dict(
     job=relation(model.Job, backref=backref('realtimetool_entry_points', uselist=True), uselist=False)
 ))
 
diff --git a/lib/galaxy/webapps/galaxy/controllers/realtime.py b/lib/galaxy/webapps/galaxy/controllers/realtime.py
index 785b18aec57..bc76340733b 100644
--- a/lib/galaxy/webapps/galaxy/controllers/realtime.py
+++ b/lib/galaxy/webapps/galaxy/controllers/realtime.py
@@ -32,7 +32,7 @@ class RealTimeToolEntryPointListGrid(grids.Grid):
 
     use_panels = True
     title = "Available InteractiveTools"
-    model_class = model.RealTimeToolEntryPoint
+    model_class = model.InteractiveToolEntryPoint
     default_filter = {"name": "All"}
     default_sort_key = "-update_time"
     columns = [
@@ -76,7 +76,7 @@ class RealTime(BaseUIController):
                     ids = [ids]
                 for entry_point_id in ids:
                     entry_point_id = self.decode_id(entry_point_id)
-                    entry_point = trans.sa_session.query(trans.app.model.RealTimeToolEntryPoint).get(entry_point_id)
+                    entry_point = trans.sa_session.query(trans.app.model.InteractiveToolEntryPoint).get(entry_point_id)
                     if trans.app.realtime_manager.can_access_entry_point(trans, entry_point):
                         eps.append(entry_point)
             if eps:

From aea9ed2a219f35ac8b94d11b51ca52ec73cdcb48 Mon Sep 17 00:00:00 2001
From: Daniel Blankenberg <dan.blankenberg@gmail.com>
Date: Wed, 21 Aug 2019 10:07:14 -0400
Subject: [PATCH 114/114] Fix get_hex for uuid4 in python3

---
 lib/galaxy/model/__init__.py                   | 2 +-
 lib/galaxy/tool_util/deps/container_classes.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 19f5aeffa45..04dc46ef509 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -1499,7 +1499,7 @@ class InteractiveToolEntryPoint(Dictifiable, RepresentById):
         self.job = job
         self.name = name
         if not token:
-            token = uuid4().get_hex()
+            token = uuid4().hex
         self.token = token
         self.tool_port = tool_port
         self.host = host
diff --git a/lib/galaxy/tool_util/deps/container_classes.py b/lib/galaxy/tool_util/deps/container_classes.py
index 458d76c26a3..abca59e8b7b 100644
--- a/lib/galaxy/tool_util/deps/container_classes.py
+++ b/lib/galaxy/tool_util/deps/container_classes.py
@@ -68,7 +68,7 @@ class Container(object):
         self.destination_info = destination_info
         self.job_info = job_info
         self.container_description = container_description
-        self.container_name = container_name or uuid4().get_hex()
+        self.container_name = container_name or uuid4().hex
         self.container_info = {}
 
     def prop(self, name, default):