Merge branch 'release_20.01' into dev

client/galaxy/scripts/mvc/ui/ui-select-default.js

@@ -211,18 +211,17 @@ var View = Backbone.View.extend({
                     return `
                     ${_.escape(result.text)}
                     <div>
-                        ${_.reduce(
-                            filteredTags.slice(0, 5),
-                            (memo, tag) => {
-                                const tagColors = keyedColorScheme(tag.slice(5));
-                                return `${memo}&nbsp;<div style="background-color: ${tagColors.primary}; color: ${
-                                    tagColors.contrasting
-                                }; border: 1px solid ${
-                                    tagColors.darker
-                                }" class="badge badge-primary badge-tags">${_.escape(tag)}</div>`;
-                            },
-                            ""
-                        )}
+                        ${filteredTags.slice(0, 5).reduce((memo, tag) => {
+                            const tagColors = keyedColorScheme(tag);
+                            const isNametag = tag.indexOf("name:") === 0;
+                            const tagDisplayText = isNametag ? `#${tag.slice(5)}` : tag;
+                            const styleBlock = isNametag
+                                ? `style="background-color: ${tagColors.primary}; color: ${tagColors.contrasting}; border: 1px solid ${tagColors.darker}"`
+                                : "";
+                            return `${memo}&nbsp;<div ${styleBlock} class="badge badge-primary badge-tags">${_.escape(
+                                tagDisplayText
+                            )}</div>`;
+                        }, "")}
                         ${extraTagWarning}
                    </div>`;
                 }

lib/galaxy/tool_util/deps/mulled/mulled_build.py

@@ -129,7 +129,7 @@ def get_affected_packages(args):
     recipes_dir = args.recipes_dir
     hours = args.diff_hours
     cmd = ['git', 'log', '--diff-filter=ACMRTUXB', '--name-only', '--pretty=""', '--since="%s hours ago"' % hours]
-    changed_files = subprocess.check_output(cmd, cwd=recipes_dir).strip().split('\n')
+    changed_files = unicodify(subprocess.check_output(cmd, cwd=recipes_dir)).splitlines()
     pkg_list = {x for x in changed_files if x.startswith('recipes/') and x.endswith('meta.yaml')}
     for pkg in pkg_list:
         if pkg and os.path.exists(os.path.join(recipes_dir, pkg)):

lib/galaxy/tool_util/deps/mulled/mulled_update_singularity_containers.py

@@ -1,36 +1,36 @@
 #!/usr/bin/env python
 
 import argparse
-import os
 import subprocess
+import tempfile
 from glob import glob
 from subprocess import check_output
 
+from galaxy.util import unicodify
 from .get_tests import hashed_test_search, main_test_search
 
 
 def get_list_from_file(filename):
-    r"""
+    """
     Returns a list of containers stored in a file (one on each line)
     """
-    return [n for n in open(filename).read().split('\n') if n != '']  # if blank lines are in the file empty strings must be removed
+    with open(filename) as fh:
+        return [_ for _ in fh.read().splitlines() if _]  # if blank lines are in the file empty strings must be removed
 
 
 def docker_to_singularity(container, installation, filepath, no_sudo=False):
     """
     Convert docker to singularity container.
     """
-
+    cmd = [installation, 'build', '/'.join((filepath, container)), 'docker://quay.io/biocontainers/' + container]
     try:
         if no_sudo:
-            check_output("%s build %s/%s docker://quay.io/biocontainers/%s" % (installation, filepath, container, container), stderr=subprocess.STDOUT, shell=True)
+            check_output(cmd, stderr=subprocess.STDOUT)
         else:
-            check_output("sudo %s build %s/%s docker://quay.io/biocontainers/%s && sudo rm -rf /root/.singularity/docker/" % (installation, filepath, container, container), stderr=subprocess.STDOUT, shell=True)
+            check_output(cmd.insert(0, 'sudo'), stderr=subprocess.STDOUT)
+            check_output(['sudo', 'rm', '-rf', '/root/.singularity/docker/'], stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError as e:
-        error_info = {'code': e.returncode, 'cmd': e.cmd, 'out': e.output}
-        return error_info
-    else:
-        return None
+        raise Exception("Docker to Singularity conversion failed.\nOutput was:\n%" % unicodify(e.output))
 
 
 def singularity_container_test(tests, installation, filepath):
@@ -40,48 +40,44 @@ def singularity_container_test(tests, installation, filepath):
     test_results = {'passed': [], 'failed': [], 'notest': []}
 
     # create a 'sanitised home' directory in which the containers may be mounted - see http://singularity.lbl.gov/faq#solution-1-specify-the-home-to-mount
-    os.mkdir("/tmp/sing_home")
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        for container, test in tests.items():
+            if 'commands' not in test and 'imports' not in test:
+                test_results['notest'].append(container)
 
-    for container, test in tests.items():
-        if 'commands' not in test and 'imports' not in test:
-            test_results['notest'].append(container)
+            else:
+                exec_command = [installation, 'exec', '-H', tmpdirname, '/'.join((filepath, container))]
+                test_passed = True
+                errors = []
+                if test.get('commands', False):
+                    for test_command in test['commands']:
+                        test_command = test_command.replace('$PREFIX', '/usr/local/')
+                        test_command = test_command.replace('${PREFIX}', '/usr/local/')
+                        test_command = test_command.replace('$R ', 'Rscript ')
 
-        else:
-            test_passed = True
-            errors = []
-            if test.get('commands', False):
-                for command in test['commands']:
-                    command = command.replace('$PREFIX', '/usr/local/')
-                    command = command.replace('${PREFIX}', '/usr/local/')
-                    command = command.replace('$R ', 'Rscript ')
-
-                    try:
-                        check_output("%s exec -H /tmp/sing_home %s/%s bash -c \"%s\"" % (
-                            installation, filepath, container, command), stderr=subprocess.STDOUT, shell=True)
-                    except subprocess.CalledProcessError:
                         try:
-                            check_output("%s exec -H /tmp/sing_home %s/%s %s" % (
-                                installation, filepath, container, command), stderr=subprocess.STDOUT, shell=True)
+                            check_output(exec_command.extend(['bash', '-c', test_command]), stderr=subprocess.STDOUT)
+                        except subprocess.CalledProcessError:
+                            try:
+                                check_output(exec_command.append(test_command), stderr=subprocess.STDOUT)
+                            except subprocess.CalledProcessError as e:
+                                errors.append(
+                                    {'command': test_command, 'output': unicodify(e.output)})
+                                test_passed = False
+
+                if test.get('imports', False):
+                    for imp in test['imports']:
+                        try:
+                            check_output(exec_command.extend([test['import_lang'], 'import ' + imp]), stderr=subprocess.STDOUT)
                         except subprocess.CalledProcessError as e:
-                            errors.append(
-                                {'command': command, 'output': e.output})
+                            errors.append({'import': imp, 'output': unicodify(e.output)})
                             test_passed = False
 
-            if test.get('imports', False):
-                for imp in test['imports']:
-                    try:
-                        check_output("%s exec -H /tmp/sing_home %s/%s %s 'import %s'" % (installation, filepath,
-                                                                                         container, test['import_lang'], imp), stderr=subprocess.STDOUT, shell=True)
-                    except subprocess.CalledProcessError as e:
-                        errors.append({'import': imp, 'output': e.output})
-                        test_passed = False
-
-            if test_passed:
-                test_results['passed'].append(container)
-            else:
-                test['errors'] = errors
-                test_results['failed'].append(test)
-    os.rmdir("/tmp/sing_home")
+                if test_passed:
+                    test_results['passed'].append(container)
+                else:
+                    test['errors'] = errors
+                    test_results['failed'].append(test)
     return test_results
 
 

lib/galaxy/webapps/reports/controllers/system.py

@@ -8,6 +8,7 @@ from sqlalchemy import and_, desc, false, null, true
 from sqlalchemy.orm import eagerload
 
 from galaxy import model, util
+from galaxy.util import unicodify
 from galaxy.webapps.base.controller import BaseUIController, web
 
 log = logging.getLogger(__name__)
@@ -151,9 +152,9 @@ class System(BaseUIController):
     def get_disk_usage(self, file_path):
         is_sym_link = os.path.islink(file_path)
         file_system = disk_size = disk_used = disk_avail = disk_cap_pct = mount = None
-        df_output = subprocess.check_output(['df', '-h', file_path])
+        df_output = unicodify(subprocess.check_output(['df', '-h', file_path]))
 
-        for df_line in df_output:
+        for df_line in df_output.splitlines():
             df_line = df_line.strip()
             if df_line:
                 df_line = df_line.lower()

lib/tool_shed/util/hg_util.py

@@ -32,7 +32,7 @@ def add_changeset(repo_path, path_to_filename_in_archive):
     except Exception as e:
         error_message = "Error adding '%s' to repository: %s" % (path_to_filename_in_archive, unicodify(e))
         if isinstance(e, subprocess.CalledProcessError):
-            error_message += "\nOutput was:\n%s" % e.output
+            error_message += "\nOutput was:\n%s" % unicodify(e.output)
         raise Exception(error_message)
 
 

test/integration/test_kubernetes_runner.py

@@ -11,6 +11,7 @@ import time
 
 import pytest
 
+from galaxy.util import unicodify
 from galaxy_test.base.populators import skip_without_tool
 from galaxy_test.driver import integration_util
 from .test_containerized_jobs import MulledJobTestCases
@@ -235,7 +236,8 @@ class BaseKubernetesIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase, M
                 time.sleep(1)
                 max_tries -= 1
 
-            status = json.loads(subprocess.check_output(['kubectl', 'get', 'job', external_id, '-o', 'json']))
+            output = unicodify(subprocess.check_output(['kubectl', 'get', 'job', external_id, '-o', 'json']))
+            status = json.loads(output)
             assert status['status']['active'] == 1
 
             delete_response = self.dataset_populator.cancel_job(job_dict["id"])
@@ -246,7 +248,7 @@ class BaseKubernetesIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase, M
             # The default job config removes jobs, didn't find a better way to check that the job doesn't exist anymore
             with pytest.raises(subprocess.CalledProcessError) as excinfo:
                 subprocess.check_output(['kubectl', 'get', 'job', external_id, '-o', 'json'], stderr=subprocess.STDOUT)
-            assert "not found" in excinfo.value.output.decode()
+            assert "not found" in unicodify(excinfo.value.output)
 
     @skip_without_tool('job_properties')
     def test_exit_code_127(self):
@@ -297,7 +299,8 @@ class BaseKubernetesIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase, M
         job_dict = running_response["jobs"][0]
         job = self.galaxy_interactor.get("jobs/%s" % job_dict['id'], admin=True).json()
         external_id = job['external_id']
-        status = json.loads(subprocess.check_output(['kubectl', 'get', 'job', external_id, '-o', 'json']))
+        output = unicodify(subprocess.check_output(['kubectl', 'get', 'job', external_id, '-o', 'json']))
+        status = json.loads(output)
         assert 'active' not in status['status']
 
     @skip_without_tool('create_2')

test/integration/test_scripts.py

@@ -182,7 +182,7 @@ class ScriptsIntegrationTestCase(integration_util.IntegrationTestCase):
             return unicodify(subprocess.check_output(cmd, cwd=cwd, env=clean_env))
         except Exception as e:
             if isinstance(e, subprocess.CalledProcessError):
-                raise Exception("%s\nOutput was:\n%s" % (unicodify(e), e.output))
+                raise Exception("%s\nOutput was:\n%s" % (unicodify(e), unicodify(e.output)))
             raise
 
     def write_config_file(self):

test/unit/tool_util/mulled/test_mulled_update_singularity_containers.py

@@ -9,7 +9,6 @@ from galaxy.util import which
 from ..util import external_dependency_management
 
 
-@external_dependency_management
 def test_get_list_from_file():
     test_dir = tempfile.mkdtemp()
     try:
@@ -25,8 +24,7 @@ def test_get_list_from_file():
 @pytest.mark.skipif(not which('singularity'), reason="requires singularity but singularity not on PATH")
 def test_docker_to_singularity(tmp_path):
     tmp_dir = str(tmp_path)
-    errors = docker_to_singularity('abundancebin:1.0.1--0', 'singularity', tmp_dir, no_sudo=True)
-    assert errors is None
+    docker_to_singularity('abundancebin:1.0.1--0', 'singularity', tmp_dir, no_sudo=True)
     assert tmp_path.joinpath('abundancebin:1.0.1--0').exists()
 
 

tools/stats/grouping.py

@@ -117,7 +117,7 @@ def main():
     try:
         subprocess.check_output(command_line, stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError as e:
-        stop_err("Sorting input dataset resulted in error: %s: %s" % (e.returncode, e.output))
+        stop_err("Sorting input dataset resulted in error: %s: %s" % (e.returncode, e.output.decode()))
 
     def is_new_item(line):
         try: