Merge remote-tracking branch 'upstream/dev' into hashtags

.travis.yml

@@ -1,3 +1,4 @@
+sudo: false
 language: python
 python: 2.7
 os:
@@ -24,9 +25,13 @@ matrix:
   allow_failures:
   - env: TOX_ENV=py27-lint-imports
 
+addons:
+  apt:
+    packages:
+      - libxml2-utils
+
 install:
   - pip install tox
-  - if [ "$TOX_ENV" == "validate-test-tools" ]; then sudo apt-get install libxml2-utils; fi
   - if [ "$TOX_ENV" == "qunit" ]; then bash -c 'cd test/qunit && npm install'; fi
   - if [ "$TOX_ENV" == "first_startup" ]; then bash -c "bash scripts/common_startup.sh && wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0127.sqlite && mv db_gx_rev_0127.sqlite database/universe.sqlite && bash manage_db.sh -c ./config/galaxy.ini.sample upgrade"; fi
 

client/galaxy/scripts/mvc/ui/ui-select-content.js

@@ -11,7 +11,7 @@ var Configurations = {
         { src: 'hdca', icon: 'fa-folder-o', tooltip: 'Dataset collection',   multiple: false,   batch: Batch.LINKED     } ],
     data_multiple: [
         { src: 'hda',  icon: 'fa-files-o',  tooltip: 'Multiple datasets',    multiple: true,    batch: Batch.DISABLED   },
-        { src: 'hdca', icon: 'fa-folder-o', tooltip: 'Dataset collection',   multiple: false,   batch: Batch.DISABLED   } ],
+        { src: 'hdca', icon: 'fa-folder-o', tooltip: 'Dataset collections',  multiple: true,    batch: Batch.DISABLED   } ],
     data_collection: [
         { src: 'hdca', icon: 'fa-folder-o', tooltip: 'Dataset collection',   multiple: false,   batch: Batch.DISABLED   } ],
     workflow_data: [

config/galaxy.ini.sample

@@ -591,6 +591,11 @@ nglims_config_file = tool-data/nglims.yaml
 # URL (with schema http/https) of the Galaxy instance as accessible within your
 # local network - if specified used as a default by pulsar file staging and
 # Jupyter Docker container for communicating back with Galaxy via the API.
+#
+# If you are attempting to setup GIEs on Mac OS X with Docker for Mac - this
+# should likely be the IP address of your machine on the virtualbox network (vboxnet0)
+# setup for the Docker host VM. This can found by running ifconfig and using the
+# IP address of the network vboxnet0.
 #galaxy_infrastructure_url = http://localhost:8080
 
 # If the above URL cannot be determined ahead of time in dynamic environments

config/plugins/interactive_environments/rstudio/config/allowed_images.yml.sample

@@ -6,7 +6,7 @@
 # appropriate `apt-get/pip install` statements.
 ---
 -
-    image: erasche/docker-rstudio-notebook:16.10
+    image: erasche/docker-rstudio-notebook:17.01
     description: |
         If you've ever done R analysis, you probably used RStudio. This
         familiar R analysis software suite will let you explore your datasets

config/plugins/interactive_environments/rstudio/config/rstudio.ini.sample

@@ -20,7 +20,7 @@ password_auth = True
 # The image argument was moved to "allowed_images.yml.sample"
 
 # Additional arguments that are passed to the `docker run` command.
-#command_inject = --sig-proxy=true -e DEBUG=false
+#command_inject = --sig-proxy=true -e DEBUG=false -e DEFAULT_CONTAINER_RUNTIME=120
 
 # URL to access the Galaxy API with from the spawn Docker containter, if empty
 # this falls back to galaxy.ini's galaxy_infrastructure_url and finally to the

config/plugins/interactive_environments/rstudio/static/js/rstudio.js

@@ -14,11 +14,11 @@ function message_failed_connection(){
  * @param {String} notebook_access_url: the URL embeded in the page and loaded
  *
  */
-function load_notebook(notebook_login_url, notebook_access_url, notebook_pubkey_url, username){
+function load_notebook(notebook_login_url, notebook_access_url, notebook_pubkey_url){
     // Test notebook_login_url for accessibility, executing the login+load function whenever
     // we've successfully connected to the IE.
     test_ie_availability(notebook_pubkey_url, function(){
-        var payload = username + "\n" + ie_password;
+        var payload = "rstudio\nrstudio";
         $.ajax({
             type: 'GET',
             url: notebook_pubkey_url,
@@ -32,7 +32,6 @@ function load_notebook(notebook_login_url, notebook_access_url, notebook_pubkey_
                 console.log("Found " + exp +" and " + mod);
                 var rsa = new RSAKey();
                 rsa.setPublic(mod, exp);
-                console.log("Encrypting '" + username + "', '" + ie_password + "'");
                 var enc_hex = rsa.encrypt(payload);
                 var encrypted = hex2b64(enc_hex);
                 console.log("E: " + encrypted);
@@ -58,7 +57,10 @@ function load_notebook(notebook_login_url, notebook_access_url, notebook_pubkey_
                         append_notebook(notebook_access_url);
                     },
                     error: function(jqxhr, status, error){
-                        message_failed_connection();
+                        // TODO: Uncomment when rstudio / nginx play nicely and
+                        // don't return http when we asked for HTTPS, thereby
+                        // issuing a mixed-content warning.
+                        //message_failed_connection();
                         // Do we want to try and load the notebook anyway? Just in case?
                         append_notebook(notebook_access_url);
                     }

config/plugins/interactive_environments/rstudio/templates/rstudio.mako

@@ -1,4 +1,5 @@
-<%namespace file="ie.mako" name="ie"/>
+<%namespace name="ie" file="ie.mako"/>
+
 <%
 import os
 import shutil
@@ -7,46 +8,42 @@ import time
 # Sets ID and sets up a lot of other variables
 ie_request.load_deploy_config()
 ie_request.attr.docker_port = 80
-# Create tempdir in galaxy
-temp_dir = ie_request.temp_dir
 PASSWORD = "rstudio"
 USERNAME = "rstudio"
 # Then override it again
 ie_request.notebook_pw = "rstudio"
 
-# Did the user give us an RData file?
-if hda.datatype.__class__.__name__ == "RData":
-    shutil.copy( hda.file_name, os.path.join(temp_dir, '.RData') )
+DATASET_HID = hda.hid
 
+# Add all environment variables collected from Galaxy's IE infrastructure
 ie_request.launch(
     image=trans.request.params.get('image_tag', None),
     additional_ids=trans.request.params.get('additional_dataset_ids', None),
     env_override={
         'notebook_username': USERNAME,
         'notebook_password': PASSWORD,
+        'dataset_hid': DATASET_HID,
     }
 )
 
 ## General IE specific
 # Access URLs for the notebook from within galaxy.
-# TODO: Make this work without pointing directly to IE. Currently does not work
-# through proxy.
-notebook_pubkey_url = ie_request.url_template('${PROXY_URL}/rstudio/auth-public-key')
 notebook_access_url = ie_request.url_template('${PROXY_URL}/rstudio/')
-notebook_login_url =  ie_request.url_template('${PROXY_URL}/rstudio/auth-do-sign-in')
+notebook_login_url = ie_request.url_template('${PROXY_URL}/rstudio/auth-do-sign-in')
+notebook_pubkey_url = ie_request.url_template('${PROXY_URL}/rstudio/auth-public-key')
 
 %>
 <html>
 <head>
 ${ ie.load_default_js() }
 </head>
-<body style="margin:0px">
+<body style="margin: 0px">
+
 <script type="text/javascript">
 ${ ie.default_javascript_variables() }
 var notebook_login_url = '${ notebook_login_url }';
 var notebook_access_url = '${ notebook_access_url }';
 var notebook_pubkey_url = '${ notebook_pubkey_url }';
-var notebook_username = '${ USERNAME }';
 require.config({
     baseUrl: app_root,
     paths: {
@@ -65,11 +62,11 @@ requirejs([
     'plugin/rstudio'
 ], function(){
     load_when_ready(ie_readiness_url, function(){
-        load_notebook(notebook_login_url, notebook_access_url, notebook_pubkey_url, "${ USERNAME }");
+        load_notebook(notebook_login_url, notebook_access_url, notebook_pubkey_url);
     });
 });
 </script>
-<div id="main">
+<div id="main" width="100%" height="100%">
 </div>
 </body>
 </html>

lib/galaxy/model/__init__.py

@@ -3512,6 +3512,14 @@ class DatasetCollectionElement( object, Dictifiable ):
         else:
             return element_object
 
+    @property
+    def dataset_instances( self ):
+        element_object = self.element_object
+        if isinstance( element_object, DatasetCollection ):
+            return element_object.dataset_instances
+        else:
+            return [element_object]
+
     def copy_to_collection( self, collection, destination=None, element_destination=None ):
         element_object = self.element_object
         if element_destination:

lib/galaxy/tools/__init__.py

@@ -153,6 +153,8 @@ GALAXY_LIB_TOOLS_UNVERSIONED = [
 # with the version at which they were fixed.
 GALAXY_LIB_TOOLS_VERSIONED = {
     "sam_to_bam": LooseVersion("1.1.3"),
+    "PEsortedSAM2readprofile": LooseVersion("1.1.1"),
+    "fetchflank": LooseVersion("1.0.1"),
 }
 
 
@@ -2310,9 +2312,10 @@ class DatabaseOperationTool( Tool ):
         for input_dataset in input_datasets.values():
             check_dataset_instance( input_dataset )
 
-        for input_dataset_collection in input_dataset_collections.values():
-            if not input_dataset_collection.collection.populated:
-                raise ToolInputsNotReadyException()
+        for input_dataset_collection_pairs in input_dataset_collections.values():
+            for input_dataset_collection, is_mapped in input_dataset_collection_pairs:
+                if not input_dataset_collection.collection.populated:
+                    raise ToolInputsNotReadyException()
 
             map( check_dataset_instance, input_dataset_collection.dataset_instances )
 

lib/galaxy/tools/actions/model_operations.py

@@ -12,8 +12,12 @@ log = logging.getLogger( __name__ )
 
 class ModelOperationToolAction( DefaultToolAction ):
 
-    def check_inputs_ready( self, tool, trans, incoming, history ):
-        history, inp_data, inp_dataset_collections = self._collect_inputs(tool, trans, incoming, history)
+    def check_inputs_ready( self, tool, trans, incoming, history, execution_cache=None ):
+        if execution_cache is None:
+            execution_cache = ToolExecutionCache(trans)
+
+        current_user_roles = execution_cache.current_user_roles
+        history, inp_data, inp_dataset_collections = self._collect_inputs(tool, trans, incoming, history, current_user_roles)
 
         tool.check_inputs_ready( inp_data, inp_dataset_collections )
 

lib/galaxy/tools/execute.py

@@ -48,7 +48,7 @@ def execute( trans, tool, param_combinations, history, rerun_remap_job_id=None,
     burst_at = getattr( config, 'tool_submission_burst_at', 10 )
     burst_threads = getattr( config, 'tool_submission_burst_threads', 1 )
 
-    tool_action = tool.action
+    tool_action = tool.tool_action
     if hasattr( tool_action, "check_inputs_ready" ):
         for params in execution_tracker.param_combinations:
             # This will throw an exception if the tool is not ready.

lib/galaxy/web/base/interactive_environments.py

@@ -6,7 +6,9 @@ import random
 import stat
 import tempfile
 import uuid
+
 from subprocess import PIPE, Popen
+from sys import platform as _platform
 
 import yaml
 
@@ -18,6 +20,8 @@ from galaxy.util import string_as_bool_or_none
 from galaxy.util.bunch import Bunch
 
 
+IS_OS_X = _platform == "darwin"
+
 CONTAINER_NAME_PREFIX = 'gie_'
 
 log = logging.getLogger(__name__)
@@ -50,7 +54,8 @@ class InteractiveEnvironmentRequest(object):
         self.load_container_interface()
 
         self.attr.docker_hostname = self.attr.viz_config.get("docker", "docker_hostname")
-        self.attr.docker_connect_port = int(self.attr.viz_config.get("docker", "docker_connect_port")) or None
+        raw_docker_connect_port = self.attr.viz_config.get("docker", "docker_connect_port")
+        self.attr.docker_connect_port = int(raw_docker_connect_port) if raw_docker_connect_port else None
 
         # Generate per-request passwords the IE plugin can use to configure
         # the destination container.
@@ -438,6 +443,10 @@ class InteractiveEnvironmentRequest(object):
             raise Exception("Attempting to launch disallowed image! %s not in list of allowed images [%s]"
                             % (image, ', '.join(self.allowed_images)))
 
+        # Do not allow a None volumes
+        if not volumes:
+            volumes = []
+
         if additional_ids is not None:
             volumes += self._idsToVolumes(additional_ids)
 
@@ -485,7 +494,10 @@ class InteractiveEnvironmentRequest(object):
         inspect_data = inspect_data[0]
         if 'Node' in inspect_data:
             return inspect_data['Node']['IP']
-        elif self.attr.docker_hostname == "localhost":
+        elif self.attr.docker_hostname == "localhost" and not IS_OS_X:
+            # If this is on Docker of Mac OS X that Gateway will be an
+            # IP address only available in the Docker host VM - so we
+            # stick with localhost.
             return inspect_data['NetworkSettings']['Gateway']
         else:
             return self.attr.docker_hostname

lib/galaxy/web/framework/middleware/remoteuser.py

@@ -69,9 +69,13 @@ class RemoteUser( object ):
             assert self.remote_user_header not in environ
             environ[ self.remote_user_header ] = self.single_user
 
-        # Apache sets REMOTE_USER to the string '(null)' when using the
-        # Rewrite* method for passing REMOTE_USER and a user is
-        # un-authenticated.  Any other possible values need to go here as well.
+        if environ.get(self.remote_user_header, '').startswith('(null)'):
+            # Throw away garbage headers.
+            # Apache sets REMOTE_USER to the string '(null)' when using the
+            # Rewrite* method for passing REMOTE_USER and a user is not authenticated.
+            # Any other possible values need to go here as well.
+            log.debug("Discarding invalid remote user header %s:%s.", self.remote_user_header, environ.get(self.remote_user_header, None))
+            environ.pop(self.remote_user_header)
         if self.remote_user_header in environ:
             # process remote user with configuration options.
             if self.normalize_remote_user_email:
@@ -126,10 +130,7 @@ class RemoteUser( object ):
                 """
                 return self.error( start_response, title, message )
 
-        # Apache sets REMOTE_USER to the string '(null)' when using the
-        # Rewrite* method for passing REMOTE_USER and a user is
-        # un-authenticated.  Any other possible values need to go here as well.
-        if not environ.get(self.remote_user_header, '(null)').startswith('(null)'):
+        if environ.get(self.remote_user_header, None):
             if not environ[ self.remote_user_header ].count( '@' ):
                 if self.maildomain is not None:
                     environ[ self.remote_user_header ] += '@' + self.maildomain

lib/galaxy/web/framework/webapp.py

@@ -385,7 +385,13 @@ class GalaxyWebTransaction( base.DefaultWebTransaction,
         elif secure_id:
             # API authentication via active session
             # Associate user using existing session
-            self._ensure_valid_session( session_cookie )
+            # This will throw an exception under remote auth with anon users.
+            try:
+                self._ensure_valid_session( session_cookie )
+            except Exception:
+                log.exception("Exception during Session-based API authentication, this was most likely an attempt to use an anonymous cookie under remote authentication (so, no user), which we don't support.")
+                self.user = None
+                self.galaxy_session = None
         else:
             # Anonymous API interaction -- anything but @expose_api_anonymous will fail past here.
             self.user = None
@@ -435,17 +441,13 @@ class GalaxyWebTransaction( base.DefaultWebTransaction,
         # cases won't have a cookie set above, so we need to to check some
         # things now.
         if self.app.config.use_remote_user:
-            # If this is an api request, and they've passed a key, we let this go.
-            assert self.app.config.remote_user_header in self.environ, \
-                "use_remote_user is set but %s header was not provided" % self.app.config.remote_user_header
-            remote_user_email = self.environ[ self.app.config.remote_user_header ]
+            remote_user_email = self.environ.get( self.app.config.remote_user_header, None)
             if galaxy_session:
-                # An existing session, make sure correct association exists
-                if galaxy_session.user is None:
+                if remote_user_email and galaxy_session.user is None:
                     # No user, associate
                     galaxy_session.user = self.get_or_create_remote_user( remote_user_email )
                     galaxy_session_requires_flush = True
-                elif (not remote_user_email.startswith('(null)') and  # Apache does this, see remoteuser.py
+                elif (remote_user_email and
                       (galaxy_session.user.email != remote_user_email) and
                       ((not self.app.config.allow_user_impersonation) or
                        (remote_user_email not in self.app.config.admin_users_list))):
@@ -456,9 +458,11 @@ class GalaxyWebTransaction( base.DefaultWebTransaction,
                     user_for_new_session = self.get_or_create_remote_user( remote_user_email )
                     log.warning( "User logged in as '%s' externally, but has a cookie as '%s' invalidating session",
                                  remote_user_email, galaxy_session.user.email )
-            else:
+            elif remote_user_email:
                 # No session exists, get/create user for new session
                 user_for_new_session = self.get_or_create_remote_user( remote_user_email )
+            if ((galaxy_session and galaxy_session.user is None) and user_for_new_session is None):
+                raise Exception("Remote Authentication Failure - user is unknown and/or not supplied.")
         else:
             if galaxy_session is not None and galaxy_session.user and galaxy_session.user.external:
                 # Remote user support is not enabled, but there is an existing

lib/galaxy/web/proxy/js/package.json

@@ -14,6 +14,6 @@
     "eventemitter3": "0.1.6",
     "http-proxy": "1.6.0",
     "commander": "~2.2",
-    "sqlite3": "3.1.3"
+    "sqlite3": "3.1.8"
   }
 }

test/api/test_workflows.py

@@ -895,6 +895,50 @@ steps:
         content = self.dataset_populator.get_history_dataset_content( history_id )
         self.assertEqual(content.strip(), "samp1\t10.0\nsamp2\t20.0\nsamp1\t20.0\nsamp2\t40.0")
 
+    def test_filter_failed_mapping( self ):
+        history_id = self.dataset_populator.new_history()
+        summary = self._run_jobs("""
+class: GalaxyWorkflow
+inputs:
+  - type: collection
+    label: input_c
+steps:
+  - label: mixed_collection
+    tool_id: exit_code_from_file
+    state:
+       input:
+         $link: input_c
+
+  - label: filtered_collection
+    tool_id: "__FILTER_FAILED_DATASETS__"
+    state:
+      input:
+        $link: mixed_collection#out_file1
+
+  - tool_id: cat1
+    state:
+      input1:
+        $link: filtered_collection
+test_data:
+  input_c:
+    type: list
+    elements:
+      - identifier: i1
+        content: "0"
+      - identifier: i2
+        content: "1"
+""", history_id=history_id, wait=True, assert_ok=False)
+        jobs = summary.jobs
+
+        def filter_jobs_by_tool(tool_id):
+            return [j for j in summary.jobs if j["tool_id"] == tool_id]
+
+        assert len(filter_jobs_by_tool("upload1")) == 2, jobs
+        assert len(filter_jobs_by_tool("exit_code_from_file")) == 2, jobs
+        assert len(filter_jobs_by_tool("__FILTER_FAILED_DATASETS__")) == 1, jobs
+        # Follow proves one job was filtered out of the result of cat1
+        assert len(filter_jobs_by_tool("cat1")) == 1, jobs
+
     def test_workflow_request( self ):
         workflow = self.workflow_populator.load_workflow( name="test_for_queue" )
         workflow_request, history_id = self._setup_workflow_run( workflow )

test/qunit/tests/ui_tests.js

@@ -698,11 +698,11 @@ define([ 'test-app', 'mvc/ui/ui-misc', 'mvc/ui/ui-select-content', 'mvc/ui/ui-dr
                 firstvalue      : 'id0',
                 firstlabel      : 'hid0: name0',
                 firstmultiple   : true,
-                totalmultiple   : 1,
+                totalmultiple   : 2,
                 lastvalue       : 'id2',
                 lastlabel       : 'hid2: name2',
                 lastlength      : 3,
-                lastmultiple    : false });
+                lastmultiple    : true });
 
         select.model.set( 'multiple', false );
         select.model.set( 'type', 'data_collection' );