Merge pull request #1403 from erasche/gie-generic

Generic GIE Launcher, GIE Image Chooser, multiple datasets

.gitignore

@@ -65,6 +65,8 @@ object_store_conf.xml
 job_metrics_conf.xml
 workflow_schedulers_conf.xml
 config/*
+config/plugins/interactive_environments/**/*.ini
+!config/plugins
 static/welcome.html.*
 static/welcome.html
 

client/galaxy/scripts/layout/menu.js

@@ -93,7 +93,12 @@ var Collection = Backbone.Collection.extend({
                     title   : 'Saved Visualizations',
                     url     : 'visualization/list',
                     target  : '_frame'
-            }]
+                },{
+                    title   : 'Interactive Environments',
+                    url     : 'visualization/gie_list',
+                    target  : 'galaxy_main'
+                }
+            ]
         });
 
         //

config/plugins/interactive_environments/jupyter/config/allowed_images.yml.sample

@@ -0,0 +1,14 @@
+# This file lists acceptable images to allow runing.
+#
+# This allows you, the admin, to create multiple flavours
+# for your users to run. E.g. maybe you need a geosciences flavour,
+# you can create the image based on our default image and add the
+# appropriate `apt-get/pip install` statements.
+---
+-
+    image: bgruening/docker-jupyter-notebook:16.01
+    description: |
+        The Jupyter notebook is the next iteration of IPython, allowing
+        analysis in many different languages. This image features the Python,
+        R, Julia, Haskell, Bash kernels and many scientific analysis stacks for
+        each.

config/plugins/interactive_environments/jupyter/config/jupyter.ini.sample

@@ -15,8 +15,7 @@
 # usual group/sudo changes.
 #command = docker {docker_args}
 
-# The docker image name that should be started.
-image = bgruening/docker-jupyter-notebook:16.01
+# The image argument was moved to "allowed_images.yml.sample"
 
 # Additional arguments that are passed to the `docker run` command.
 command_inject = --sig-proxy=true -e DEBUG=false -e DEFAULT_CONTAINER_RUNTIME=120

config/plugins/interactive_environments/jupyter/templates/jupyter.mako

@@ -24,10 +24,14 @@ else:
     DATASET_HID = None
 
 # Add all environment variables collected from Galaxy's IE infrastructure
-ie_request.launch(env_override={
-    'notebook_password': PASSWORD,
-    'dataset_hid': DATASET_HID,
-})
+ie_request.launch(
+    image=trans.request.params.get('image_tag', None),
+    additional_ids=trans.request.params.get('additional_dataset_ids', None),
+    env_override={
+        'notebook_password': PASSWORD,
+        'dataset_hid': DATASET_HID,
+    }
+)
 
 ## General IE specific
 # Access URLs for the notebook from within galaxy.

config/plugins/interactive_environments/rstudio/config/allowed_images.yml.sample

@@ -0,0 +1,14 @@
+# This file lists acceptable images to allow runing.
+#
+# This allows you, the admin, to create multiple flavours
+# for your users to run. E.g. maybe you need a geosciences flavour,
+# you can create the image based on our default image and add the
+# appropriate `apt-get/pip install` statements.
+---
+-
+    image: erasche/docker-rstudio-notebook:15.10
+    description: |
+        If you've ever done R analysis, you probably used RStudio. This
+        familiar R analysis software suite will let you explore your datasets
+        in depth. Comes with ggplot2, RODBC, maps, shinyapps, knitr, LaTeX,
+        bioconductor, cummeRbund, and many more pre-installed packages.

config/plugins/interactive_environments/rstudio/config/rstudio.ini.sample

@@ -17,8 +17,7 @@ password_auth = True
 # usual group/sudo changes.
 #command = docker {docker_args}
 
-# The docker image name that should be started.
-image = erasche/docker-rstudio-notebook:15.10
+# The image argument was moved to "allowed_images.yml.sample"
 
 # Additional arguments that are passed to the `docker run` command.
 #command_inject = --sig-proxy=true -e DEBUG=false

config/plugins/interactive_environments/rstudio/templates/rstudio.mako

@@ -16,10 +16,14 @@ USERNAME = "galaxy"
 if hda.datatype.__class__.__name__ == "RData":
     shutil.copy( hda.file_name, os.path.join(temp_dir, '.RData') )
 
-ie_request.launch(env_override={
-    'notebook_username': USERNAME,
-    'notebook_password': PASSWORD,
-})
+ie_request.launch(
+    image=trans.request.params.get('image_tag', None),
+    additional_ids=trans.request.params.get('additional_dataset_ids', None),
+    env_override={
+        'notebook_username': USERNAME,
+        'notebook_password': PASSWORD,
+    }
+)
 
 ## General IE specific
 # Access URLs for the notebook from within galaxy.

lib/galaxy/config.py

@@ -451,6 +451,7 @@ class Configuration( object ):
         self.visualization_plugins_directory = kwargs.get(
             'visualization_plugins_directory', 'config/plugins/visualizations' )
         ie_dirs = kwargs.get( 'interactive_environment_plugins_directory', None )
+        self.gie_dirs = [d.strip() for d in (ie_dirs.split(",") if ie_dirs else [])]
         if ie_dirs and not self.visualization_plugins_directory:
             self.visualization_plugins_directory = ie_dirs
         elif ie_dirs:

lib/galaxy/web/base/interactive_environments.py

@@ -2,13 +2,14 @@ import ConfigParser
 
 import os
 import json
+import yaml
 import stat
 import random
 import tempfile
 from subprocess import Popen, PIPE
 
 from galaxy.util.bunch import Bunch
-from galaxy import web
+from galaxy import web, model
 from galaxy.managers import api_keys
 from galaxy.tools.deps.docker_util import DockerVolume
 
@@ -39,6 +40,7 @@ class InteractiveEnvironmentRequest(object):
         self.attr.HOST = trans.request.host.rsplit(':', 1)[0]
 
         self.load_deploy_config()
+        self.load_allowed_images()
         self.attr.docker_hostname = self.attr.viz_config.get("docker", "docker_hostname")
 
         # Generate per-request passwords the IE plugin can use to configure
@@ -69,6 +71,29 @@ class InteractiveEnvironmentRequest(object):
         else:
             self.attr.proxy_prefix = ''
 
+    def load_allowed_images(self):
+        if os.path.exists(os.path.join(self.attr.our_config_dir, 'allowed_images.yml')):
+            fn = os.path.join(self.attr.our_config_dir, 'allowed_images.yml')
+        elif os.path.exists(os.path.join(self.attr.our_config_dir, 'allowed_images.yml.sample')):
+            fn = os.path.join(self.attr.our_config_dir, 'allowed_images.yml.sample')
+        else:
+            # If we don't have an allowed images, then we fall back to image
+            # name specified in the .ini file
+            try:
+                self.allowed_images = [self.attr.viz_config.image]
+                self.default_image = self.attr.viz_config.image
+                return
+            except AttributeError:
+                raise Exception("[{0}] Could not find allowed_images.yml, or image tag in {0}.ini file for ".format(self.attr.viz_id))
+
+        with open(fn, 'r') as handle:
+            self.allowed_images = [x['image'] for x in yaml.load(handle)]
+
+            if len(self.allowed_images) == 0:
+                raise Exception("No allowed images specified for " + self.attr.viz_id)
+
+            self.default_image = self.allowed_images[0]
+
     def load_deploy_config(self, default_dict={}):
         # For backwards compat, any new variables added to the base .ini file
         # will need to be recorded here. The ConfigParser doesn't provide a
@@ -178,7 +203,7 @@ class InteractiveEnvironmentRequest(object):
     def volume(self, host_path, container_path, **kwds):
         return DockerVolume(host_path, container_path, **kwds)
 
-    def docker_cmd(self, env_override={}, volumes=[]):
+    def docker_cmd(self, image, env_override={}, volumes=[]):
         """
             Generate and return the docker command to execute
         """
@@ -201,13 +226,64 @@ class InteractiveEnvironmentRequest(object):
             environment=env_str,
             import_volume_str=import_volume_str,
             volume_str=volume_str,
-            image=self.attr.viz_config.get("docker", "image")
+            image=image,
         )
         return command
 
-    def launch(self, raw_cmd=None, env_override={}, volumes=[]):
+    def _idsToVolumes(self, ids):
+        if len(ids.strip()) == 0:
+            return []
+
+        # They come as a comma separated list
+        ids = ids.split(',')
+
+        # Next we need to turn these into volumes
+        volumes = []
+        for id in ids:
+            decoded_id = self.trans.security.decode_id(id)
+            dataset = self.trans.sa_session.query(model.HistoryDatasetAssociation).get(decoded_id)
+            # TODO: do we need to check if the user has access?
+            volumes.append(self.volume(dataset.get_file_name(), '/import/[{0}] {1}.{2}'.format(dataset.id, dataset.name, dataset.ext)))
+        return volumes
+
+    def launch(self, image=None, additional_ids=None, raw_cmd=None, env_override={}, volumes=[]):
+        """Launch a docker image.
+
+        :type image: str
+        :param image: Optional image name. If not provided, self.default_image
+                      is used, which is the first image listed in the
+                      allowed_images.yml{,.sample} file.
+
+        :type additional_ids: str
+        :param additional_ids: comma separated list of encoded HDA IDs. These
+                               are transformed into Volumes and added to that
+                               argument
+
+        :type raw_cmd: str
+        :param raw_cmd: raw docker command. Usually generated with self.docker_cmd()
+
+        :type env_override: dict
+        :param env_override: dictionary of environment variables to add.
+
+        :type volumes: list of galaxy.tools.deps.docker_util.DockerVolume
+        :param volumes: dictionary of docker volume mounts
+
+        """
+        if image is None:
+            image = self.default_image
+
+        if image not in self.allowed_images:
+            # Now that we're allowing users to specify images, we need to ensure that they aren't
+            # requesting images we have not specifically allowed.
+            raise Exception("Attempting to launch disallowed image! %s not in list of allowed images [%s]"
+                            % (image, ', '.join(self.allowed_images)))
+
+        if additional_ids is not None:
+            volumes += self._idsToVolumes(additional_ids)
+
         if raw_cmd is None:
-            raw_cmd = self.docker_cmd(env_override=env_override, volumes=volumes)
+            raw_cmd = self.docker_cmd(image, env_override=env_override, volumes=volumes)
+
         log.info("Starting docker container for IE {0} with command [{1}]".format(
             self.attr.viz_id,
             raw_cmd

lib/galaxy/webapps/galaxy/controllers/visualization.py

@@ -22,6 +22,8 @@ from galaxy.web.base.controller import BaseUIController, SharableMixin, UsesVisu
 from galaxy.web.framework.helpers import grids, time_ago
 
 from .library import LibraryListGrid
+import os
+import yaml
 
 log = logging.getLogger( __name__ )
 
@@ -1013,6 +1015,45 @@ class VisualizationController( BaseUIController, SharableMixin, UsesVisualizatio
             }
         return trans.fill_template_mako( "visualization/phyloviz.mako", data=data, config=config )
 
+    @web.expose
+    @web.require_login( "run Galaxy Interactive Environments" )
+    def gie_list( self, trans, **kwargs ):
+        if not hasattr( self, 'gie_image_map' ):
+            self.gie_image_map = {}
+
+            for gie_dir in self.app.config.gie_dirs:
+                gie_list = os.listdir( gie_dir )
+                for gie in gie_list:
+                    gie_path = os.path.join(gie_dir, gie)
+
+                    if not os.path.isdir(gie_path):
+                        continue
+
+                    if not os.path.exists(self._gie_config_dir(gie_path)):
+                        continue
+
+                    if os.path.exists( self._gie_config_dir( gie_path, 'allowed_images.yml' ) ):
+                        image_file = self._gie_config_dir( gie_path, 'allowed_images.yml' )
+                    elif os.path.exists( self._gie_config_dir( gie_path, 'allowed_images.yml.sample' ) ):
+                        image_file = self._gie_config_dir( gie_path, 'allowed_images.yml.sample' )
+                    else:
+                        continue
+
+                    with open( image_file, 'r' ) as handle:
+                        self.gie_image_map[gie] = yaml.load( handle )
+
+        return trans.fill_template_mako(
+            "visualization/gie.mako",
+            gie_image_map=self.gie_image_map,
+            history=trans.get_history(),
+        )
+
+    def _gie_config_dir(self, gie_path, *args):
+        nargs = [gie_path, 'config']
+        if len(args) > 0:
+            nargs += args
+        return os.path.join(*nargs)
+
     @web.json
     def bookmarks_from_dataset( self, trans, hda_id=None, ldda_id=None ):
         if hda_id:

templates/webapps/galaxy/visualization/gie.mako

@@ -0,0 +1,157 @@
+<%inherit file="/webapps/galaxy/base_panels.mako"/>
+
+<%def name="init()">
+<%
+    self.has_left_panel=False
+    self.has_right_panel=False
+    self.active_view="visualization"
+    self.message_box_visible=False
+%>
+</%def>
+
+<%def name="center_panel()">
+
+    ${h.js('libs/jquery/select2')}
+
+    <div style="overflow: auto; height: 100%;">
+        <div class="page-container" style="padding: 10px;">
+            <div class="col-md-8 col-md-offset 2 col-xs-12">
+                <div class="row">
+                    <h1>Galaxy Interactive Environment Launcher</h1>
+                    <p>
+                        Galaxy Interactive Environments (GIEs) launch embedded,
+                        dockerized versions of popular data analysis suites such as
+                        Jupyter and RStudio, right from within Galaxy. They allow you
+                        to dynamically interact with your data, right on the server. No
+                        more uploading and downloading between various platforms just
+                        to get your work done!
+                        <br />
+                        <a href="https://docs.galaxyproject.org/en/master/admin/interactive_environments.html">Admin Docs</a>
+                    </p>
+                    <form id='launcher' action="NONE" method="GET">
+
+                        <table class="table table-striped">
+                            <tr>
+                                <td>GIE: </td>
+                                <td>
+                                    <span id="image_name" style="width: 400px" />
+                                </td>
+                            </tr>
+                            <tr>
+                                <td>Image: </td>
+                                <td>
+                                    <span id="image_tag" style="width:400px" />
+                                    <input id="image_tag_hidden" type="hidden" name="image_tag" value="NONE" />
+                                    <p id="image_desc">
+                                    </p>
+                                </td>
+                            </tr>
+                            <tr>
+                                <td>Datasets: </td>
+                                <td>
+                                    <span id="additional_datasets" style="width:400px"></span>
+                                    <input id="dataset_id" name="dataset_id" type="hidden">
+                                    <input id="additional_dataset_ids" name="additional_dataset_ids" type="hidden">
+                                </td>
+                            </tr>
+                        </table>
+                        <input type="submit" class="button" value="Launch" disabled="">
+                    </form>
+                </div>
+            </div>
+
+
+        </div>
+    </div>
+
+<script type="text/javascript">
+$(document).ready(function(){
+    var gie_image_map = {
+        % for image_name in gie_image_map.keys():
+            "${image_name}": [
+                % for image in gie_image_map[image_name]:
+                {
+                    id: "${image['image']}",
+                    text: "${image['image']}",
+                    extra: "${image['description'].replace('\n', ' ')}",
+                },
+                % endfor
+            ],
+        % endfor
+    }
+
+    var images = [
+        % for image_name in gie_image_map.keys():
+        {
+            id: "${image_name}",
+            text: "${image_name}"
+        },
+        % endfor
+    ]
+
+    var datasets = [
+        % for hda in history.visible_datasets[::-1]:
+            { id: "${ trans.security.encode_id(hda.id) }", text: "${ hda.hid } : ${ hda.name }" },
+        % endfor
+    ]
+
+    $("#additional_datasets").select2({
+        multiple: true,
+        data: datasets
+    }).on('change', function(e){
+        ds_ids = $("#additional_datasets").val().split(',');
+        if(ds_ids.length < 1){
+            $('input[type="submit"]').attr('disabled', '');
+        } else {
+            $("#dataset_id").val(ds_ids[0])
+
+            // In a perfect world the controller would just support a single
+            // parameter passing a list of dataset IDs.
+            //
+            // We aren't in that world (yet).
+            if(ds_ids.length > 1){
+                $("#additional_dataset_ids").val(ds_ids.slice(1).join(","))
+            }else{
+                $("#additional_dataset_ids").val("")
+            }
+            $('input[type="submit"]').removeAttr('disabled');
+        }
+    })
+
+    function formatter(v){
+        if(!v.id) return v.text;
+        return "<b>" + v.id + "</b><p>" + v.extra + "</p>"
+    }
+
+    $('#image_name').select2({
+        placeholder: "Select Image",
+        data: images
+    }).on('change', function(e){
+        // Get the versions for this image name
+        image_versions = gie_image_map[e.val]
+        // Update the action
+        $("#launcher").attr("action", "../plugins/interactive_environments/" + e.val + "/show")
+        // Update the hidden input
+        $("#image_name_hidden").val(e.val)
+        // Set disabled if they switch image family without updating image.
+        $('input[type="submit"]').attr('disabled', '');
+
+        // Create our select2 appropriately
+        image_tags = $("#image_tag").select2({
+            placholder: "Image Version",
+            formatResult: formatter,
+            formatSelection: formatter,
+            escapeMarkup: function(m) { return m; },
+            data: image_versions
+        }).on('change', function(e2){
+            // Inner actions, update the hidden input
+            $("#image_tag_hidden").val(e2.val)
+            // Enable the button
+            if($("additional_datasets").length > 0){
+                $('input[type="submit"]').removeAttr('disabled');
+            }
+        })
+    })
+})
+</script>
+</%def>