opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #790 from erasche/gie-docker-launch

Browse Source 
Refactor GIE Launching to use -P
This commit is contained in:
John Chilton
2015-09-25 16:05:35 +01:00
parent 0fefff07b3 5f15b0d94c
commit 6d3ea7ee00
6 changed files with 122 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/galaxy.ini.sample
+6
View File
@@ -574,6 +574,12 @@ nglims_config_file = tool-data/nglims.yaml
# nodejs to wrap connections in SSL).
#dynamic_proxy_external_proxy=False
# Additionally, when the dynamic proxy is proxied by an upstream server, you'll
# want to specify a prefixed URL so both Galaxy and the proxy reside under the
# same path that your cookies are under. This will result in a url like
# https://FQDN/galaxy-prefix/gie_proxy for proxying
#dynamic_proxy_prefix=gie_proxy
# -- Logging and Debugging
# If True, Galaxy will attempt to configure a simple root logger if a
config/plugins/interactive_environments/ipython/templates/ipython.mako
+5 -7
View File
@@ -33,18 +33,16 @@ if hda.datatype.__class__.__name__ != "Ipynb":
else:
shutil.copy( hda.file_name, empty_nb_path )
## General IE specific
# Access URLs for the notebook from within galaxy.
notebook_access_url = ie_request.url_template('${PROXY_URL}/ipython/${PORT}/notebooks/ipython_galaxy_notebook.ipynb')
notebook_login_url = ie_request.url_template('${PROXY_URL}/ipython/${PORT}/login?next=%2Fipython%2F${PORT}%2Ftree')
# Add all environment variables collected from Galaxy's IE infrastructure
ie_request.launch(env_override={
'notebook_password': PASSWORD,
})
## General IE specific
# Access URLs for the notebook from within galaxy.
notebook_access_url = ie_request.url_template('${PROXY_URL}/ipython/notebooks/ipython_galaxy_notebook.ipynb')
notebook_login_url = ie_request.url_template('${PROXY_URL}/ipython/login?next=${PROXY_PREFIX}%2Fipython%2Ftree')
%>
<html>
<head>
config/plugins/interactive_environments/rstudio/templates/rstudio.mako
+9 -9
View File
@@ -12,14 +12,6 @@ temp_dir = ie_request.temp_dir
PASSWORD = ie_request.notebook_pw
USERNAME = "galaxy"
## General IE specific
# Access URLs for the notebook from within galaxy.
# TODO: Make this work without pointing directly to IE. Currently does not work
# through proxy.
notebook_pubkey_url = ie_request.url_template('${PROXY_URL}/rstudio/${PORT}/auth-public-key')
notebook_access_url = ie_request.url_template('${PROXY_URL}/rstudio/${PORT}/')
notebook_login_url = ie_request.url_template('${PROXY_URL}/rstudio/${PORT}/auth-do-sign-in')
# Did the user give us an RData file?
if hda.datatype.__class__.__name__ == "RData":
shutil.copy( hda.file_name, os.path.join(temp_dir, '.RData') )
@@ -27,8 +19,16 @@ if hda.datatype.__class__.__name__ == "RData":
ie_request.launch(env_override={
'notebook_username': USERNAME,
'notebook_password': PASSWORD,
'cors_origin': ie_request.attr.proxy_url,
})
## General IE specific
# Access URLs for the notebook from within galaxy.
# TODO: Make this work without pointing directly to IE. Currently does not work
# through proxy.
notebook_pubkey_url = ie_request.url_template('${PROXY_URL}/rstudio/auth-public-key')
notebook_access_url = ie_request.url_template('${PROXY_URL}/rstudio/')
notebook_login_url = ie_request.url_template('${PROXY_URL}/rstudio/auth-do-sign-in')
%>
<html>
<head>
lib/galaxy/config.py
+1
View File
@@ -443,6 +443,7 @@ class Configuration( object ):
self.dynamic_proxy_bind_port = int( kwargs.get( "dynamic_proxy_bind_port", "8800" ) )
self.dynamic_proxy_bind_ip = kwargs.get( "dynamic_proxy_bind_ip", "0.0.0.0" )
self.dynamic_proxy_external_proxy = string_as_bool( kwargs.get( "dynamic_proxy_external_proxy", "False" ) )
self.dynamic_proxy_prefix = kwargs.get( "dynamic_proxy_prefix", "gie_proxy" )
# Default chunk size for chunkable datatypes -- 64k
self.display_chunk_size = int( kwargs.get( 'display_chunk_size', 65536) )
lib/galaxy/web/base/interactive_environments.py
+96 -25
View File
@@ -1,6 +1,7 @@
import ConfigParser
import os
import json
import stat
import random
import tempfile
@@ -40,11 +41,6 @@ class InteractiveEnviornmentRequest(object):
self.load_deploy_config()
self.attr.docker_hostname = self.attr.viz_config.get("docker", "docker_hostname")
self.attr.proxy_request = trans.app.proxy_manager.setup_proxy(
trans, host=self.attr.docker_hostname
)
self.attr.proxy_url = self.attr.proxy_request[ 'proxy_url' ]
self.attr.PORT = self.attr.proxy_request[ 'proxied_port' ]
# Generate per-request passwords the IE plugin can use to configure
# the destination container.
@@ -62,6 +58,14 @@ class InteractiveEnviornmentRequest(object):
log.error( "Could not change permissions of tmpdir %s" % self.temp_dir )
# continue anyway
# This duplicates the logic in the proxy manager
if self.attr.galaxy_config.dynamic_proxy_external_proxy:
self.attr.proxy_prefix = '%s/%s' % (
self.attr.galaxy_config.cookie_path,
self.attr.galaxy_config.dynamic_proxy_prefix)
else:
self.attr.proxy_prefix = ''
def load_deploy_config(self, default_dict={}):
# For backwards compat, any new variables added to the base .ini file
# will need to be recorded here. The ConfigParser doesn't provide a
@@ -107,8 +111,10 @@ class InteractiveEnviornmentRequest(object):
'history_id': self.attr.history_id,
'api_key': api_key,
'remote_host': request.remote_addr,
'docker_port': self.attr.PORT,
# DOCKER_PORT is NO LONGER AVAILABLE. All IEs must update.
'cors_origin': request.host_url,
'user_email': self.trans.user.email,
'proxy_prefix': self.attr.proxy_prefix,
}
if self.attr.viz_config.has_option("docker", "galaxy_url"):
@@ -143,36 +149,27 @@ class InteractiveEnviornmentRequest(object):
return "false"
def url_template(self, url_template):
"""Process a URL template
There are several variables accessible to the user:
- ${PROXY_URL} will be replaced with the dynamically create proxy's url
- ${PROXY_PREFIX} will be replaced with the prefix that may occur
"""
Process a URL template
There are several variables accessible to the user:
- ${PROXY_URL} will be replaced with dynamically create proxy
- ${PORT} will be replaced with the port the docker image is attached to
"""
# Figure out our substitutions
# Next several lines for older style replacements (not used with Galaxy dynamic
# proxy)
if self.attr.SSL_URLS:
protocol = 'https'
else:
protocol = 'http'
if not self.attr.APACHE_URLS:
# If they are not using apache URLs, that implies there's a port attached to the host
# string, thus we replace just the first instance of host that we see.
url_template = url_template.replace('${HOST}', '${HOST}:${PORT}', 1)
url_template = url_template.replace('${PROTO}', protocol) \
.replace('${HOST}', self.attr.HOST)
# Only the following replacements are used with Galaxy dynamic proxy
# URLs
url = url_template.replace('${PROXY_URL}', str(self.attr.proxy_url)) \
.replace('${PORT}', str(self.attr.PORT))
.replace('${PROXY_PREFIX}', str(self.attr.proxy_prefix.replace('/', '%2F')))
return url
def volume(self, host_path, container_path, **kwds):
@@ -193,13 +190,11 @@ class InteractiveEnviornmentRequest(object):
# Then we format in the entire docker command in place of
# {docker_args}, so as to let the admin not worry about which args are
# getting passed
command = command.format(docker_args='run {command_inject} {environment} -d -p {port_ext}:{port_int} -v "{temp_dir}:/import/" {volume_str} {image}')
command = command.format(docker_args='run {command_inject} {environment} -d -P -v "{temp_dir}:/import/" {volume_str} {image}')
# Once that's available, we format again with all of our arguments
command = command.format(
command_inject=self.attr.viz_config.get("docker", "command_inject"),
environment=env_str,
port_ext=self.attr.PORT,
port_int=self.attr.docker_port,
temp_dir=temp_dir,
volume_str=volume_str,
image=self.attr.viz_config.get("docker", "image")
@@ -217,5 +212,81 @@ class InteractiveEnviornmentRequest(object):
stdout, stderr = p.communicate()
if p.returncode != 0 or len(stderr):
log.error( "%s\n%s" % (stdout, stderr) )
return None
else:
log.debug( "Container id: %s" % stdout)
port_mappings = self.get_proxied_ports(stdout)
if len(port_mappings) > 1:
log.warning("Don't know how to handle proxies to containers with multiple exposed ports. Arbitrarily choosing first")
elif len(port_mappings) == 0:
log.warning("No exposed ports to map! Images MUST EXPOSE")
return None
# Fetch the first port_mapping
(service, host_ip, host_port) = port_mappings[0]
# Now we configure our proxy_requst object and we manually specify
# the port to map to and ensure the proxy is available.
self.attr.proxy_request = self.trans.app.proxy_manager.setup_proxy(
self.trans,
host=self.attr.docker_hostname,
port=host_port,
proxy_prefix=self.attr.proxy_prefix,
)
# These variables then become available for use in templating URLs
self.attr.proxy_url = self.attr.proxy_request[ 'proxy_url' ]
# Commented out because it needs to be documented and visible that
# this variable was moved here. Usually would remove commented
# code, but again, needs to be clear where this went. Remove at a
# later time.
#
# PORT is no longer exposed internally. All requests are forced to
# go through the proxy we ship.
# self.attr.PORT = self.attr.proxy_request[ 'proxied_port' ]
def get_proxied_ports(self, container_id):
"""Run docker inspect on a container to figure out which ports were
mapped where.
:type container_id: str
:param container_id: a docker container ID
:returns: a list of triples containing (internal_port, external_ip,
external_port), of which the ports are probably the only
useful information.
Someday code that calls this should be refactored whenever we get
containers with multiple ports working.
"""
command = self.attr.viz_config.get("docker", "command")
command = command.format(docker_args="inspect %s" % container_id)
log.info("Inspecting docker container {0} with command [{1}]".format(
container_id,
command
))
p = Popen(command, stdout=PIPE, stderr=PIPE, close_fds=True, shell=True)
stdout, stderr = p.communicate()
if p.returncode != 0 or len(stderr):
log.error( "%s\n%s" % (stdout, stderr) )
return None
inspect_data = json.loads(stdout)
# [{
# "NetworkSettings" : {
# "Ports" : {
# "3306/tcp" : [
# {
# "HostIp" : "127.0.0.1",
# "HostPort" : "3306"
# }
# ]
mappings = []
port_mappings = inspect_data[0]['NetworkSettings']['Ports']
for port_name in port_mappings:
for binding in port_mappings[port_name]:
mappings.append((
port_name.replace('/tcp', '').replace('/udp', ''),
binding['HostIp'],
binding['HostPort']
))
return mappings
lib/galaxy/web/proxy/__init__.py
+5 -3
View File
@@ -17,7 +17,9 @@ SECURE_COOKIE = "galaxysession"
class ProxyManager(object):
def __init__( self, config ):
for option in [ "manage_dynamic_proxy", "dynamic_proxy_bind_port", "dynamic_proxy_bind_ip", "dynamic_proxy_debug", "dynamic_proxy_external_proxy" ]:
for option in ["manage_dynamic_proxy", "dynamic_proxy_bind_port",
"dynamic_proxy_bind_ip", "dynamic_proxy_debug",
"dynamic_proxy_external_proxy", "dynamic_proxy_prefix"]:
setattr( self, option, getattr( config, option ) )
self.launch_by = "node" # TODO: Support docker
if self.manage_dynamic_proxy:
@@ -29,7 +31,7 @@ class ProxyManager(object):
def shutdown( self ):
self.lazy_process.shutdown()
def setup_proxy( self, trans, host=DEFAULT_PROXY_TO_HOST, port=None ):
def setup_proxy( self, trans, host=DEFAULT_PROXY_TO_HOST, port=None, proxy_prefix="" ):
if self.manage_dynamic_proxy:
log.info("Attempting to start dynamic proxy process")
self.lazy_process.start_process()
@@ -46,7 +48,7 @@ class ProxyManager(object):
if not self.dynamic_proxy_external_proxy:
proxy_url = '%s://%s:%d' % (scheme, host, self.dynamic_proxy_bind_port)
else:
proxy_url = '%s://%s' % (scheme, host)
proxy_url = '%s://%s%s' % (scheme, host, proxy_prefix)
return {
'proxy_url': proxy_url,
'proxied_port': proxy_requests.port,