mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
John Chilton
@@ -785,8 +785,8 @@ nglims_config_file = tool-data/nglims.yaml
|
||||
# notebooks in Galaxy histories. This process may allow users to execute
|
||||
# arbitrary code or serve arbitrary HTML. If enabled, IPython must be
|
||||
# available and on Galaxy's PATH, to do this run
|
||||
# `pip install jinja2 pygments ipython` in Galaxy's virtualenv.
|
||||
#trust_ipython_notebook_conversion = False
|
||||
# `pip install jinja2 pygments jupyter` in Galaxy's virtualenv.
|
||||
#trust_jupyter_notebook_conversion = False
|
||||
|
||||
# Debug enables access to various config options useful for development and
|
||||
# debugging: use_lint, use_profile, use_printdebug and use_interactive. It
|
||||
|
||||
@@ -229,8 +229,8 @@ Docker host with
|
||||
$ docker -H tcp://gx-docker:4243 run -it busybox sh
|
||||
|
||||
So far so good! Now we need to configure Galaxy to use our new Docker host
|
||||
to start the Interactive Environments. For that we need to edit the IPython GIE
|
||||
configuration, ``ipython.ini`` to use our custom docker host
|
||||
to start the Interactive Environments. For that we need to edit the Jupyter GIE
|
||||
configuration, ``jupyter.ini`` to use our custom docker host
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
inkscape:version="0.48.5 r10040"
|
||||
sodipodi:docname="diagram.svg">
|
||||
<title
|
||||
id="title5754">Galaxy IPython Graphic</title>
|
||||
id="title5754">Galaxy Jupyter Graphic</title>
|
||||
<defs
|
||||
id="defs4">
|
||||
<marker
|
||||
@@ -481,7 +481,7 @@
|
||||
<dc:format>image/svg+xml</dc:format>
|
||||
<dc:type
|
||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
||||
<dc:title>Galaxy IPython Graphic</dc:title>
|
||||
<dc:title>Galaxy Jupyter Graphic</dc:title>
|
||||
<dc:creator>
|
||||
<cc:Agent>
|
||||
<dc:title>Björn Grüning, Torsten Houwaart, Eric Rasche</dc:title>
|
||||
@@ -523,7 +523,7 @@
|
||||
sodipodi:role="line"
|
||||
id="tspan3805"
|
||||
x="142.18896"
|
||||
y="291.41959">Launch IPython</tspan><tspan
|
||||
y="291.41959">Launch Jupyter</tspan><tspan
|
||||
sodipodi:role="line"
|
||||
x="142.18896"
|
||||
y="303.91959"
|
||||
@@ -701,7 +701,7 @@
|
||||
sodipodi:role="line"
|
||||
id="tspan5090-9-4"
|
||||
x="214.56726"
|
||||
y="364.25571">ipython_galaxy_notebook.ipynb</tspan></text>
|
||||
y="364.25571">jupyter_galaxy_notebook.ipynb</tspan></text>
|
||||
<rect
|
||||
style="fill:#ffffff;stroke:#000000;stroke-width:0.625;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0.25"
|
||||
id="rect5068-6"
|
||||
@@ -777,7 +777,7 @@
|
||||
y="306.04703"
|
||||
x="211.61868"
|
||||
id="tspan5302"
|
||||
sodipodi:role="line">IPython + scipy stack installed</tspan></text>
|
||||
sodipodi:role="line">Jupyter + scipy stack installed</tspan></text>
|
||||
<rect
|
||||
ry="0"
|
||||
y="281.74686"
|
||||
@@ -817,7 +817,7 @@
|
||||
y="378.99741"
|
||||
x="215.05746"
|
||||
sodipodi:role="line"
|
||||
style="font-size:6px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;writing-mode:lr-tb;text-anchor:start;fill:#000000;stroke:none;font-family:Sans;-inkscape-font-specification:Sans">IPython Webservice </tspan></text>
|
||||
style="font-size:6px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;writing-mode:lr-tb;text-anchor:start;fill:#000000;stroke:none;font-family:Sans;-inkscape-font-specification:Sans">Jupyter Webservice </tspan></text>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-size:6px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;font-family:Courier 10 Pitch;-inkscape-font-specification:Courier 10 Pitch"
|
||||
|
||||
|
Before Width: | Height: | Size: 175 KiB After Width: | Height: | Size: 175 KiB |
@@ -14,21 +14,21 @@ plugins, for those familiar with developing those
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ tree $GALAXY_ROOT/config/plugins/interactive_environments/ipython/
|
||||
config/plugins/interactive_environments/ipython/
|
||||
$ tree $GALAXY_ROOT/config/plugins/interactive_environments/jupyter
|
||||
config/plugins/interactive_environments/jupyter
|
||||
├── config
|
||||
│ ├── ipython.ini
|
||||
│ ├── ipython.ini.sample
|
||||
│ └── ipython.xml
|
||||
│ ├── jupyter.ini
|
||||
│ ├── jupyter.ini.sample
|
||||
│ └── jupyter.xml
|
||||
├── static
|
||||
│ └── js
|
||||
│ └── ipython.js
|
||||
│ └── jupyter.js
|
||||
└── templates
|
||||
├── ipython.mako
|
||||
├── jupyter.mako
|
||||
└── notebook.ipynb
|
||||
|
||||
We'll use the variable ``{gie}`` to stand in for the name of your GIE. It
|
||||
should match ``[a-z]+``, like ``ipython`` or ``rstudio``. Here you can see the
|
||||
should match ``[a-z]+``, like ``jupyter`` or ``rstudio``. Here you can see the
|
||||
``config/`` directory with a ``{gie}.ini.sample`` providing docker and image
|
||||
configuration, and then ``{gie}.xml`` which outlines that it is a GIE.
|
||||
|
||||
@@ -37,7 +37,7 @@ you are actively developing a GIE, you'll need to restart Galaxy after adding
|
||||
any resources to that file, before they can be accessed in the browser.
|
||||
|
||||
Lastly, and most importantly, there's the templates folder. This normally just
|
||||
contains ``{gie}.mako``, however the IPython file needs an extra template file.
|
||||
contains ``{gie}.mako``, however the Jupyter file needs an extra template file.
|
||||
|
||||
First Steps, Configuration
|
||||
--------------------------
|
||||
@@ -186,9 +186,9 @@ We'll continue appending to our ``helloworld.mako`` the HTML code that's actuall
|
||||
</body>
|
||||
</html>
|
||||
|
||||
We've glossed over some of the features of this file, but most IEs do a significant amount of "magic" in the top half of the mako template. For instance, the IPython notebook:
|
||||
We've glossed over some of the features of this file, but most IEs do a significant amount of "magic" in the top half of the mako template. For instance, the Jupyter notebook:
|
||||
|
||||
- If the user is trying to run the IPython GIE Visualization on an existing notebook in their history, then that gets loaded into the docker container via the temp directory and set as the default notebook
|
||||
- If the user is trying to run the Jupyter GIE Visualization on an existing notebook in their history, then that gets loaded into the docker container via the temp directory and set as the default notebook
|
||||
- Otherwise a default notebook is built for the user.
|
||||
|
||||
The RStudio notebook:
|
||||
@@ -252,7 +252,7 @@ This function is very short. Historically, the GIE process involved a complex da
|
||||
|
||||
Since the NodeJS proxy takes care of authentication/authorization, we can
|
||||
reduce the helloworld ``_handle_notebook_loading`` function to a simple
|
||||
``append_notebook`` call. You may wish to look at the IPython and RStudio GIEs
|
||||
``append_notebook`` call. You may wish to look at the Jupyter and RStudio GIEs
|
||||
for examples of the complex things that can be done at every step.
|
||||
|
||||
The GIE Container
|
||||
@@ -301,7 +301,7 @@ Here's an example Dockerfile for our helloworld container
|
||||
ADD ./startup.sh /startup.sh
|
||||
ADD ./monitor_traffic.sh /monitor_traffic.sh
|
||||
|
||||
# /import will be the universal mount-point for IPython
|
||||
# /import will be the universal mount-point for Jupyter
|
||||
# The Galaxy instance can copy in data that needs to be present to the
|
||||
# container
|
||||
RUN mkdir /import
|
||||
@@ -426,11 +426,11 @@ As you can see, a LOT is going on! We'll break it down further:
|
||||
|
||||
- The user's API key is provided, allowing you to access datasets and
|
||||
submit jobs on their behalf. If you have an environment like
|
||||
IPython/RStudio, it is **highly recommended** that you provide some magic
|
||||
Jupyter/RStudio, it is **highly recommended** that you provide some magic
|
||||
by which the user can use their API key without embedding it in the
|
||||
notebook. If you do embed it somehow in a document that gets saved to
|
||||
their history, anyone can impersonate that user if they get a hold of it.
|
||||
In the IPython GIE we have a variable that just runs
|
||||
In the Jupyter GIE we have a variable that just runs
|
||||
``os.environ.get('API_KEY')`` to avoid embedding it in their notebook.
|
||||
- A CORS Origin is provided for very strict servers, but it may be easier
|
||||
to simply void CORS requirements within the nginx proxy in your
|
||||
@@ -449,7 +449,7 @@ As you can see, a LOT is going on! We'll break it down further:
|
||||
which is a small utility script to provide API access to Galaxy to get
|
||||
and fetch data, based on those environment variables.
|
||||
- The ``HISTORY_ID`` of the current history the user is on is provided. In
|
||||
the IPython/RStudio containers, we provide a dead simple method for users
|
||||
the Jupyter/RStudio containers, we provide a dead simple method for users
|
||||
to download datasets from their current history which will be visible to
|
||||
them on the right hand side of their screen.
|
||||
- A ``PROXY_PREFIX`` is provided which should be used in the nginx conf.
|
||||
@@ -466,9 +466,9 @@ As you can see, a LOT is going on! We'll break it down further:
|
||||
Most of this information is usually required to build friendly, easy-to-use
|
||||
GIEs. One of the strong points of GIEs is their magic interaction with Galaxy.
|
||||
Here we've mounted a volume read-only, but in real life you may wish to provide
|
||||
connectivity like IPython and RStudio provide, allowing the user to load
|
||||
connectivity like Jupyter and RStudio provide, allowing the user to load
|
||||
datasets on demand for interactive analysis, and then to store analysis
|
||||
artefacts (and a log of what was done inside the container, à la IPython's
|
||||
artefacts (and a log of what was done inside the container, à la Jupyter's
|
||||
"notebooks") back to their current history.
|
||||
|
||||
If everything went well, at this point you should see a directory listing show up:
|
||||
|
||||
@@ -276,7 +276,7 @@ class Configuration( object ):
|
||||
self.sanitize_whitelist_file = resolve_path( kwargs.get( 'sanitize_whitelist_file', "config/sanitize_whitelist.txt" ), self.root )
|
||||
self.serve_xss_vulnerable_mimetypes = string_as_bool( kwargs.get( 'serve_xss_vulnerable_mimetypes', False ) )
|
||||
self.allowed_origin_hostnames = self._parse_allowed_origin_hostnames( kwargs )
|
||||
self.trust_ipython_notebook_conversion = string_as_bool( kwargs.get( 'trust_ipython_notebook_conversion', False ) )
|
||||
self.trust_jupyter_notebook_conversion = string_as_bool( kwargs.get( 'trust_jupyter_notebook_conversion', False ) )
|
||||
self.enable_old_display_applications = string_as_bool( kwargs.get( "enable_old_display_applications", "True" ) )
|
||||
self.brand = kwargs.get( 'brand', None )
|
||||
self.welcome_url = kwargs.get( 'welcome_url', '/static/welcome.html' )
|
||||
@@ -706,7 +706,7 @@ class Configuration( object ):
|
||||
return resolve_path( path, self.root )
|
||||
|
||||
def guess_galaxy_port(self):
|
||||
# Code derived from IPython work ie.mako
|
||||
# Code derived from Jupyter work ie.mako
|
||||
config = configparser.SafeConfigParser({'port': '8080'})
|
||||
if self.config_file:
|
||||
config.read( self.config_file )
|
||||
|
||||
@@ -108,7 +108,7 @@ class Ipynb( Json ):
|
||||
def set_peek( self, dataset, is_multi_byte=False ):
|
||||
if not dataset.dataset.purged:
|
||||
dataset.peek = get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte )
|
||||
dataset.blurb = "IPython Notebook"
|
||||
dataset.blurb = "Jupyter Notebook"
|
||||
else:
|
||||
dataset.peek = 'file does not exist'
|
||||
dataset.blurb = 'file purged from disc'
|
||||
@@ -129,7 +129,7 @@ class Ipynb( Json ):
|
||||
|
||||
def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, **kwd):
|
||||
config = trans.app.config
|
||||
trust = getattr( config, 'trust_ipython_notebook_conversion', False )
|
||||
trust = getattr( config, 'trust_jupyter_notebook_conversion', False )
|
||||
if trust:
|
||||
return self._display_data_trusted(trans, dataset, preview=preview, filename=filename, to_ext=to_ext, **kwd)
|
||||
else:
|
||||
@@ -144,13 +144,13 @@ class Ipynb( Json ):
|
||||
ofilename = ofile_handle.name
|
||||
ofile_handle.close()
|
||||
try:
|
||||
cmd = 'ipython nbconvert --to html --template full %s --output %s' % (dataset.file_name, ofilename)
|
||||
cmd = 'jupyter nbconvert --to html --template full %s --output %s' % (dataset.file_name, ofilename)
|
||||
log.info("Calling command %s" % cmd)
|
||||
subprocess.call(cmd, shell=True)
|
||||
ofilename = '%s.html' % ofilename
|
||||
except:
|
||||
ofilename = dataset.file_name
|
||||
log.exception( 'Command "%s" failed. Could not convert the IPython Notebook to HTML, defaulting to plain text.' % cmd )
|
||||
log.exception( 'Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.' % cmd )
|
||||
return open( ofilename )
|
||||
|
||||
def set_meta( self, dataset, **kwd ):
|
||||
|
||||
@@ -244,7 +244,7 @@ class VisualizationPlugin( pluginframework.Plugin, ServesStaticPluginMixin, Serv
|
||||
# =============================================================================
|
||||
class InteractiveEnvironmentPlugin( VisualizationPlugin ):
|
||||
"""
|
||||
Serves web-based REPLs such as IPython and RStudio.
|
||||
Serves web-based REPLs such as Jupyter and RStudio.
|
||||
"""
|
||||
INTENV_REQUEST_FACTORY = interactive_environments.InteractiveEnvironmentRequest
|
||||
|
||||
@@ -262,7 +262,7 @@ class InteractiveEnvironmentPlugin( VisualizationPlugin ):
|
||||
# this feels hacky to me but it's what mako recommends:
|
||||
# http://docs.makotemplates.org/en/latest/runtime.html
|
||||
render_vars.update( vars={} )
|
||||
# No longer needed but being left around for a few releases as ipython-galaxy
|
||||
# No longer needed but being left around for a few releases as jupyter-galaxy
|
||||
# as an external visualization plugin is deprecated in favor of core interactive
|
||||
# environment plugin.
|
||||
if 'get_api_key' not in render_vars:
|
||||
|
||||
@@ -99,7 +99,7 @@ class VisualizationsRegistry( pluginframework.PageServingPluginManager ):
|
||||
|
||||
# default class
|
||||
plugin_class = vis_plugins.VisualizationPlugin
|
||||
# ipython, etc
|
||||
# jupyter, etc
|
||||
if config[ 'plugin_type' ] == 'interactive_environment':
|
||||
plugin_class = vis_plugins.InteractiveEnvironmentPlugin
|
||||
# js only
|
||||
|
||||
@@ -70,7 +70,7 @@ class Sentry(object):
|
||||
# remote service) and can be considered a security risk as well. For
|
||||
# multiple services running alongside Galaxy on the same host, this
|
||||
# could allow a sentry user with access to logs to impersonate a user
|
||||
# on another service. In the case of services like IPython, this can be
|
||||
# on another service. In the case of services like Jupyter, this can be
|
||||
# a serious concern as that would allow for terminal access. Furthermore,
|
||||
# very little debugging information can be gained as a result of having
|
||||
# access to all of the users cookies (including Galaxy cookies)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Have not yet gotten this to work - goal was to launch the prox in a Docker container.
|
||||
# Networking is a bit tricky though - could not get the child proxy to talk to the child
|
||||
# IPython container.
|
||||
# Jupyter container.
|
||||
|
||||
|
||||
# sudo docker build --no-cache=true -t gxproxy .
|
||||
|
||||
@@ -146,10 +146,10 @@ class VisualizationsRegistry_TestCase( unittest.TestCase ):
|
||||
def test_interactive_environ_plugin_load( self ):
|
||||
"""
|
||||
"""
|
||||
ipython_config = utility.clean_multiline_string( """\
|
||||
jupyter_config = utility.clean_multiline_string( """\
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE interactive_environment SYSTEM "../../interactive_environments.dtd">
|
||||
<interactive_environment name="IPython">
|
||||
<interactive_environment name="Jupyter">
|
||||
<data_sources>
|
||||
<data_source>
|
||||
<model_class>HistoryDatasetAssociation</model_class>
|
||||
@@ -161,15 +161,15 @@ class VisualizationsRegistry_TestCase( unittest.TestCase ):
|
||||
<params>
|
||||
<param type="dataset" var_name_in_template="hda" required="true">dataset_id</param>
|
||||
</params>
|
||||
<template>ipython.mako</template>
|
||||
<template>jupyter.mako</template>
|
||||
</interactive_environment>
|
||||
""" )
|
||||
|
||||
mock_app_dir = {
|
||||
'plugins': {
|
||||
'ipython': {
|
||||
'jupyter': {
|
||||
'config': {
|
||||
'ipython.xml': ipython_config
|
||||
'jupyter.xml': jupyter_config
|
||||
},
|
||||
'templates': {}
|
||||
},
|
||||
@@ -177,8 +177,8 @@ class VisualizationsRegistry_TestCase( unittest.TestCase ):
|
||||
}
|
||||
|
||||
# going to use a fake template here to simplify testing
|
||||
ipython_template = "${ ie_request }-${ get_api_key() }"
|
||||
mock_app_dir[ 'plugins' ][ 'ipython' ][ 'templates' ][ 'ipython.mako' ] = ipython_template
|
||||
jupyter_template = "${ ie_request }-${ get_api_key() }"
|
||||
mock_app_dir[ 'plugins' ][ 'jupyter' ][ 'templates' ][ 'jupyter.mako' ] = jupyter_template
|
||||
# so that we don't create a cached version of that fake template in the real mako caches
|
||||
# we'll set up a cache in the temp dir
|
||||
mock_app_dir[ 'caches' ] = {}
|
||||
@@ -191,26 +191,26 @@ class VisualizationsRegistry_TestCase( unittest.TestCase ):
|
||||
|
||||
# ...then start testing
|
||||
expected_plugins_path = os.path.join( mock_app_dir.root_path, 'plugins' )
|
||||
expected_plugin_names = [ 'ipython' ]
|
||||
expected_plugin_names = [ 'jupyter' ]
|
||||
|
||||
self.assertEqual( plugin_mgr.base_url, 'visualizations' )
|
||||
self.assertEqual( plugin_mgr.directories, [ expected_plugins_path ] )
|
||||
self.assertEqual( sorted(plugin_mgr.plugins.keys()), expected_plugin_names )
|
||||
|
||||
ipython = plugin_mgr.plugins[ 'ipython' ]
|
||||
config = ipython.config
|
||||
jupyter = plugin_mgr.plugins[ 'jupyter' ]
|
||||
config = jupyter.config
|
||||
|
||||
self.assertEqual( ipython.name, 'ipython' )
|
||||
self.assertEqual( jupyter.name, 'jupyter' )
|
||||
self.assertEqual( config.get( 'plugin_type' ), 'interactive_environment' )
|
||||
|
||||
# get_api_key needs a user, fill_template a trans
|
||||
user = model.User( email="blah@bler.blah", password="dockerDockerDOCKER" )
|
||||
trans = galaxy_mock.MockTrans( user=user )
|
||||
# use a mock request factory - this will be written into the filled template to show it was used
|
||||
ipython.INTENV_REQUEST_FACTORY = lambda t, p: 'mock'
|
||||
jupyter.INTENV_REQUEST_FACTORY = lambda t, p: 'mock'
|
||||
|
||||
# should return the (new) api key for the above user (see the template above)
|
||||
response = ipython._render( {}, trans=trans )
|
||||
response = jupyter._render( {}, trans=trans )
|
||||
response.strip()
|
||||
self.assertIsInstance( response, string_types )
|
||||
self.assertTrue( '-' in response )
|
||||
|
||||
Reference in New Issue
Block a user