From d5fc7d426e2d1bc519a35288735dd47cb332273d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 3 Jan 2020 17:49:18 +0100 Subject: [PATCH] Fix interactivetool python 3 support and tests - enables docker destination for interactivetool tests - fixes a wrong argument in entrypoint management - fixes the test setup that targeted a previous iteration (which was still called realtimetools) - prevents deleting the working directory before python has a chance to start the container monitor (that's the sleep command) - fixes some odd interactive tool test assertions - uses unicode arguments in key_type_token_mapping.py. uwsgi arguments are always passed as bytestrings. --- lib/galaxy/jobs/__init__.py | 2 +- lib/galaxy/managers/interactivetool.py | 2 +- lib/galaxy_ext/container_monitor/monitor.py | 2 +- lib/galaxy_test/driver/driver_util.py | 20 +++++++++++-------- .../key_type_token_mapping.py | 13 ++++++++++-- test/integration/test_containerized_jobs.py | 5 +++++ test/integration/test_interactivetools_api.py | 12 +++++------ 7 files changed, 36 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 9bb04908efd..8b9d67523ca 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -2111,7 +2111,7 @@ class JobWrapper(HasResourceParameters): "connection_configuration": container.connection_configuration, }, f) - return "(python '%s'/lib/galaxy_ext/container_monitor/monitor.py &) " % exec_dir + return "(python '%s'/lib/galaxy_ext/container_monitor/monitor.py &); sleep 1 " % exec_dir @property def user(self): diff --git a/lib/galaxy/managers/interactivetool.py b/lib/galaxy/managers/interactivetool.py index dd740f96eb1..a6504c5c587 100644 --- a/lib/galaxy/managers/interactivetool.py +++ b/lib/galaxy/managers/interactivetool.py @@ -270,7 +270,7 @@ class InteractiveToolManager(object): entry_point = trans.sa_session.query(model.InteractiveToolEntryPoint).get(entry_point_id) if self.app.interactivetool_manager.can_access_entry_point(trans, entry_point): if entry_point.active: - return self.target_if_active(entry_point) + return self.target_if_active(trans, entry_point) elif entry_point.deleted: raise exceptions.MessageException('InteractiveTool has ended. You will have to start a new one.') else: diff --git a/lib/galaxy_ext/container_monitor/monitor.py b/lib/galaxy_ext/container_monitor/monitor.py index b1e1447f1e0..86987b81e2d 100644 --- a/lib/galaxy_ext/container_monitor/monitor.py +++ b/lib/galaxy_ext/container_monitor/monitor.py @@ -22,7 +22,7 @@ def parse_ports(container_name, connection_configuration): preexec_fn=os.setpgrp) if exit_code == 0: stdout_file.seek(0) - ports_raw = stdout_file.read() + ports_raw = stdout_file.read().decode('utf-8') return ports_raw diff --git a/lib/galaxy_test/driver/driver_util.py b/lib/galaxy_test/driver/driver_util.py index f6334b762ee..1475c8669e1 100644 --- a/lib/galaxy_test/driver/driver_util.py +++ b/lib/galaxy_test/driver/driver_util.py @@ -4,6 +4,7 @@ import fcntl import logging import os import random +import re import shutil import signal import socket @@ -62,13 +63,15 @@ INSTALLED_TOOL_PANEL_CONFIGS = [ ] REALTIME_PROXY_TEMPLATE = string.Template(r""" uwsgi: - realtime_map: $tempdir/realtime_map.sqlite - python-raw: scripts/realtime/key_type_token_mapping.py - route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.$test_host:$test_port)$ goto:realtime + http-raw-body: true + interactivetools_map: $tempdir/interactivetools_map.sqlite + python-raw: scripts/interactivetools/key_type_token_mapping.py + route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)-([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(interactivetool\.$test_host:$test_port)$ goto:interactivetool route-run: goto:endendend - route-label: realtime - route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(realtime\.$test_host:$test_port)$ rpcvar:TARGET_HOST rtt_key_type_token_mapper_cached $2 $1 $3 $4 $0 5 + route-label: interactivetool + route-host: ^([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)-([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.([A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)\.(interactivetool\.$test_host:$test_port)$ rpcvar:TARGET_HOST rtt_key_type_token_mapper_cached $1 $3 $2 $4 $0 5 route-if-not: empty:${TARGET_HOST} httpdumb:${TARGET_HOST} + route: .* break:404 Not Found route-label: endendend """) @@ -729,8 +732,9 @@ def launch_uwsgi(kwargs, tempdir, prefix=DEFAULT_CONFIG_PREFIX, config_object=No enable_realtime_mapping = getattr(config_object, "enable_realtime_mapping", False) if enable_realtime_mapping: - config["galaxy"]["realtime_prefix"] = "realtime" - config["galaxy"]["realtime_map"] = os.path.join(tempdir, "realtime_map.sqlite") + config["galaxy"]["interactivetools_prefix"] = "interactivetool" + config["galaxy"]["interactivetools_map"] = os.path.join(tempdir, "interactivetools_map.sqlite") + config['galaxy']['interactivetools_enable'] = True yaml_config_path = os.path.join(tempdir, "galaxy.yml") with open(yaml_config_path, "w") as f: @@ -743,7 +747,7 @@ def launch_uwsgi(kwargs, tempdir, prefix=DEFAULT_CONFIG_PREFIX, config_object=No old_contents = f.read() with open(yaml_config_path, "w") as f: test_port = str(port) if port else r"[0-9]+" - test_host = host or "localhost" + test_host = re.escape(host) if host else "localhost" uwsgi_section = REALTIME_PROXY_TEMPLATE.safe_substitute(test_host=test_host, test_port=test_port, tempdir=tempdir) f.write(uwsgi_section) f.write(old_contents) diff --git a/scripts/interactivetools/key_type_token_mapping.py b/scripts/interactivetools/key_type_token_mapping.py index fa096dfbd60..df6dff6506a 100644 --- a/scripts/interactivetools/key_type_token_mapping.py +++ b/scripts/interactivetools/key_type_token_mapping.py @@ -6,7 +6,7 @@ from time import time import uwsgi -realtime_db_file = uwsgi.opt["interactivetools_map"] +realtime_db_file = uwsgi.opt["interactivetools_map"].decode('utf-8') db_conn = sqlite3.connect(realtime_db_file) DATABASE_TABLE_NAME = 'gxitproxy' @@ -51,6 +51,14 @@ class CacheList(): key_type_token_mapped_cache = CacheList() +def args_as_unicode(func): + def wrap_args(*args): + args = (arg.decode('utf-8') if isinstance(arg, bytes) else arg for arg in args) + return func(*args) + return wrap_args + + +@args_as_unicode def key_type_token_mapper_cached(key, key_type, token, route_extra, url, ttl): global key_type_token_mapped_cache cache_key = (key, key_type, token) @@ -58,11 +66,12 @@ def key_type_token_mapper_cached(key, key_type, token, route_extra, url, ttl): if entry is None: entry = key_type_token_mapper(key, key_type, token, route_extra, url) if entry is not None: - # Should we cache empt/not authorized entries, perhaps for shorter time? + # Should we cache empty/not authorized entries, perhaps for shorter time? key_type_token_mapped_cache.add_entry(cache_key, entry, ttl=float(ttl)) return entry +@args_as_unicode def key_type_token_mapper(key, key_type, token, route_extra, url): global db_conn # print 'key %s key_type %s token %s route_extra %s url %s\n' % (key, key_type, token, route_extra, url) diff --git a/test/integration/test_containerized_jobs.py b/test/integration/test_containerized_jobs.py index 626d63c5e63..d84f3060b8d 100644 --- a/test/integration/test_containerized_jobs.py +++ b/test/integration/test_containerized_jobs.py @@ -37,6 +37,11 @@ class ContainerizedIntegrationTestCase(integration_util.IntegrationTestCase): skip_if_container_type_unavailable(cls) super(ContainerizedIntegrationTestCase, cls).setUpClass() + @classmethod + def handle_galaxy_config_kwds(cls, config): + config["job_config_file"] = DOCKERIZED_JOB_CONFIG_FILE + disable_dependency_resolution(config) + def disable_dependency_resolution(config): # Disable tool dependency resolution. diff --git a/test/integration/test_interactivetools_api.py b/test/integration/test_interactivetools_api.py index e214e3eee80..f3ae60d4879 100644 --- a/test/integration/test_interactivetools_api.py +++ b/test/integration/test_interactivetools_api.py @@ -47,8 +47,7 @@ class InteractiveToolsIntegrationTestCase(ContainerizedIntegrationTestCase): assert isinstance(jobs, list) assert len(jobs) == 1 job0 = jobs[0] - entry_points = self.wait_on_entry_points_active(job0["id"]) - assert len(entry_points) == 2 + entry_points = self.wait_on_entry_points_active(job0["id"], expected_num=2) entry_point0 = entry_points[0] entry_point1 = entry_points[1] target0 = self.entry_point_target(entry_point0["id"]) @@ -59,19 +58,18 @@ class InteractiveToolsIntegrationTestCase(ContainerizedIntegrationTestCase): content1 = self.wait_on_proxied_content(target1) assert content1 == "moo cow\n", content1 - assert False def wait_on_proxied_content(self, target): def get_hosted_content(): try: scheme, rest = target.split("://", 1) - prefix, host_and_port = rest.split(".realtime.") - print(rest) + prefix, host_and_port = rest.split(".interactivetool.") faked_host = rest if "/" in rest: faked_host = rest.split("/", 1)[0] - response = requests.get("%s://%s" % (scheme, host_and_port), timeout=1, headers={"Host": faked_host}) - return response.content + url = "%s://%s" % (scheme, host_and_port) + response = requests.get(url, timeout=1, headers={"Host": faked_host}) + return response.text except Exception as e: print(e) return None