Merge pull request #1388 from jmchilton/py3_2

Python 3 fixes and galaxy-lib updates from planemo.

lib/galaxy/exceptions/error_codes.py

@@ -26,7 +26,7 @@ class ErrorCode( object ):
         message = entry.get("message")
         return ( name, ErrorCode( code, message ) )
 
-error_codes_json = resource_string( __name__, 'error_codes.json' )
+error_codes_json = resource_string( __name__, 'error_codes.json' ).decode( "UTF-8" )
 for entry in loads( error_codes_json ):
     name, error_code_obj = ErrorCode.from_dict( entry )
     globals()[ name ] = error_code_obj

lib/galaxy/tools/deps/commands.py

@@ -1,16 +1,49 @@
 import os
 import subprocess
+import sys as _sys
 
 
-def shell(cmds, env=None):
-    p = shell_process(cmds, env)
-    return p.wait()
+def redirecting_io(sys=_sys):
+    assert sys is not None
+    # We are redirecting standard out and standard error.
+    return not hasattr(sys.stdout, "fileno")
+
+
+def redirect_aware_commmunicate(p, sys=_sys):
+    assert sys is not None
+    out, err = p.communicate()
+    if redirecting_io(sys=sys):
+        if out:
+            sys.stdout.write(out)
+            out = None
+        if err:
+            sys.stderr.write(err)
+            err = None
+    return out, err
+
+
+def shell(cmds, env=None, **kwds):
+    sys = kwds.get("sys", _sys)
+    assert sys is not None
+    p = shell_process(cmds, env, **kwds)
+    if redirecting_io(sys=sys):
+        redirect_aware_commmunicate(p, sys=sys)
+        exit = p.returncode
+        return exit
+    else:
+        return p.wait()
 
 
 def shell_process(cmds, env=None, **kwds):
+    sys = kwds.get("sys", _sys)
     popen_kwds = dict(
         shell=True,
     )
+    if kwds.get("stdout", None) is None and redirecting_io(sys=sys):
+        popen_kwds["stdout"] = subprocess.PIPE
+    if kwds.get("stderr", None) is None and redirecting_io(sys=sys):
+        popen_kwds["stderr"] = subprocess.PIPE
+
     popen_kwds.update(**kwds)
     if env:
         new_env = os.environ.copy()
@@ -41,6 +74,16 @@ def __wait(cmds, **popen_kwds):
     return stdout
 
 
+def download_command(url, quote_url=False):
+    if quote_url:
+        url = "'%s'" % url
+    if which("wget"):
+        download_cmd = ["wget", "-q", "--recursive", "-O" "-", url]
+    else:
+        download_cmd = ["curl", "-L", url]
+    return download_cmd
+
+
 class CommandLineException(Exception):
 
     def __init__(self, command, stdout, stderr):

lib/galaxy/tools/deps/containers.py

@@ -5,6 +5,8 @@ from abc import (
 import os
 import string
 
+import six
+
 from galaxy.util import asbool
 from ..deps import docker_util
 
@@ -208,7 +210,7 @@ class DockerContainer(Container):
         # Allow destinations to explicitly set environment variables just for
         # docker container. Better approach is to set for destination and then
         # pass through only what tool needs however. (See todo in ToolInfo.)
-        for key, value in self.destination_info.iteritems():
+        for key, value in six.iteritems(self.destination_info):
             if key.startswith("docker_env_"):
                 env = key[len("docker_env_"):]
                 env_directives.append('"%s=%s"' % (env, value))

lib/galaxy/tools/lint.py

@@ -10,6 +10,7 @@ LEVEL_ERROR = "error"
 def lint_xml(tool_xml, level=LEVEL_ALL, fail_level=LEVEL_WARN, extra_modules=[], skip_types=[]):
     lint_context = LintContext(level=level, skip_types=skip_types)
     lint_xml_with(lint_context, tool_xml, extra_modules)
+
     return not lint_context.failed(fail_level)
 
 

lib/galaxy/tools/linters/inputs.py

@@ -24,34 +24,49 @@ def lint_inputs(tool_xml, lint_ctx):
         if param_type == "data":
             if "format" not in param_attrib:
                 lint_ctx.warn("Param input [%s] with no format specified - 'data' format will be assumed.", param_name)
-        # TODO: Validate type, much more...
 
-    conditional_selects = tool_xml.findall("./inputs//conditional")
-    for conditional in conditional_selects:
-        select = conditional.find('./param[@type="select"]') or []
-        boolean = conditional.find('./param[@type="boolean"]') or []
-        # Should conditionals ever not have a select?
-        if not len(select) and not len(boolean):
-            lint_ctx.warn("Conditional without <param type=\"select\" /> or <param type=\"boolean\" />")
-            continue
+        if param_type == "select":
+            dynamic_options = param.get("dynamic_options", None)
+            if dynamic_options is None:
+                dynamic_options = param.find("options")
 
-        if len(select):
-            select_options = select.findall('./option[@value]')
+            select_options = _find_with_attribute(param, 'option', 'value')
             if any(['value' not in option.attrib for option in select_options]):
                 lint_ctx.error("Option without value")
 
             select_option_ids = [option.attrib.get('value', None) for option in select_options]
-        else:
+
+            if dynamic_options is None and len(select_options) == 0:
+                message = "No options defined for select [%s]" % param_name
+                lint_ctx.warn(message)
+
+        # TODO: Validate type, much more...
+
+    conditional_selects = tool_xml.findall("./inputs//conditional")
+    for conditional in conditional_selects:
+        booleans = _find_with_attribute(conditional, "param", "type", "boolean")
+        selects = _find_with_attribute(conditional, "param", "type", "select")
+        # Should conditionals ever not have a select?
+        if not len(selects) and not len(booleans):
+            lint_ctx.warn("Conditional without <param type=\"select\" /> or <param type=\"boolean\" />")
+            continue
+
+        for select in selects:
+            select_options = _find_with_attribute(select, 'option', 'value')
+            select_option_ids = [option.attrib.get('value', None) for option in select_options]
+
+        for boolean in booleans:
             select_option_ids = [
-                boolean.attrib.get('truevalue', 'True'),
-                boolean.attrib.get('falsevalue', 'False')
+                boolean.attrib.get('truevalue', 'true'),
+                boolean.attrib.get('falsevalue', 'false')
             ]
 
         whens = conditional.findall('./when')
         if any(['value' not in when.attrib for when in whens]):
             lint_ctx.error("When without value")
 
-        when_ids = [when.attrib.get('value', None) for when in whens]
+        when_ids = [w.attrib.get('value', None) for w in whens]
+        when_ids = [i.lower() if i in ["True", "False"] else i for i in when_ids]
 
         for select_id in select_option_ids:
             if select_id not in when_ids:
@@ -82,3 +97,17 @@ def lint_repeats(tool_xml, lint_ctx):
             lint_ctx.error("Repeat does not specify name attribute.")
         if "title" not in repeat.attrib:
             lint_ctx.error("Repeat does not specify title attribute.")
+
+
+def _find_with_attribute(element, tag, attribute, test_value=None):
+    rval = []
+    for el in (element.findall('./%s' % tag) or []):
+        if attribute not in el.attrib:
+            continue
+        value = el.attrib[attribute]
+        if test_value is not None:
+            if value == test_value:
+                rval.append(el)
+        else:
+            rval.append(el)
+    return rval

lib/galaxy/tools/linters/stdio.py

@@ -39,7 +39,7 @@ def _lint_exit_code(child, lint_ctx):
 
 
 def _lint_regex(child, lint_ctx):
-    for key, value in child.attrib.iteritems():
+    for key, value in child.attrib.items():
         if key == "source":
             if value not in ["stderr", "stdout", "both"]:
                 lint_ctx.error("Unknown error code level encountered [%s]" % value)

lib/galaxy/tools/parser/output_objects.py

@@ -114,7 +114,7 @@ class ToolOutputCollection( ToolOutputBase ):
         # This line is probably not right - should verify structured_like
         # or have outputs and all outputs have name.
         if len( self.outputs ) > 1:
-            output_parts = [ToolOutputCollectionPart(self, k, v) for k, v in self.outputs.iteritems()]
+            output_parts = [ToolOutputCollectionPart(self, k, v) for k, v in self.outputs.items()]
         else:
             # either must have specified structured_like or something worse
             if self.structure.structured_like:

lib/galaxy/tools/parser/yaml.py

@@ -169,7 +169,7 @@ def _parse_test(i, test_dict):
     inputs = test_dict["inputs"]
     if _is_dict(inputs):
         new_inputs = []
-        for key, value in inputs.iteritems():
+        for key, value in inputs.items():
             new_inputs.append((key, value, {}))
         test_dict["inputs"] = new_inputs
 
@@ -177,7 +177,7 @@ def _parse_test(i, test_dict):
 
     new_outputs = []
     if _is_dict(outputs):
-        for key, value in outputs.iteritems():
+        for key, value in outputs.items():
             if _is_dict(value):
                 attributes = value
                 file = attributes.get("file")
@@ -322,6 +322,6 @@ class YamlInputSource(InputSource):
 
 
 def _ensure_has(dict, defaults):
-    for key, value in defaults.iteritems():
+    for key, value in defaults.items():
         if key not in dict:
             dict[key] = value

lib/galaxy/tools/toolbox/base.py

@@ -7,6 +7,7 @@ import tempfile
 
 from markupsafe import escape
 from six.moves.urllib.parse import urlparse
+from six import iteritems
 
 from galaxy.util.dictifiable import Dictifiable
 
@@ -456,7 +457,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
         return []
 
     def tools( self ):
-        return self._tools_by_id.iteritems()
+        return iteritems(self._tools_by_id)
 
     def dynamic_confs( self, include_migrated_tool_conf=False ):
         confs = []

lib/galaxy/tools/toolbox/panel.py

@@ -4,6 +4,7 @@ from galaxy.util.odict import odict
 from galaxy.util import bunch
 from galaxy.util.dictifiable import Dictifiable
 
+from six import iteritems
 
 panel_item_types = bunch.Bunch(
     TOOL="TOOL",
@@ -27,7 +28,7 @@ class HasPanelItems:
         """ Iterate through panel items each represented as a tuple of
         (panel_key, panel_type, panel_content).
         """
-        for panel_key, panel_value in self.panel_items().iteritems():
+        for panel_key, panel_value in iteritems(self.panel_items()):
             if panel_value is None:
                 continue
             panel_type = panel_item_types.SECTION

lib/galaxy/util/__init__.py

@@ -23,6 +23,7 @@ import time
 import tempfile
 import threading
 from six.moves.urllib import parse as urlparse
+from six import iteritems
 
 from galaxy.util import json
 from datetime import datetime
@@ -226,17 +227,17 @@ def xml_element_to_dict( elem ):
     if sub_elems:
         sub_elem_dict = dict()
         for sub_sub_elem_dict in map( xml_element_to_dict, sub_elems ):
-            for key, value in sub_sub_elem_dict.iteritems():
+            for key, value in iteritems(sub_sub_elem_dict):
                 if key not in sub_elem_dict:
                     sub_elem_dict[ key ] = []
                 sub_elem_dict[ key ].append( value )
-        for key, value in sub_elem_dict.iteritems():
+        for key, value in iteritems(sub_elem_dict):
             if len( value ) == 1:
                 rval[ elem.tag ][ key ] = value[0]
             else:
                 rval[ elem.tag ][ key ] = value
     if elem.attrib:
-        for key, value in elem.attrib.iteritems():
+        for key, value in iteritems(elem.attrib):
             rval[ elem.tag ][ "@%s" % key ] = value
 
     if elem.text:
@@ -1010,14 +1011,14 @@ def stringify_dictionary_keys( in_dict ):
     # changes unicode keys into strings, only works on top level (does not recurse)
     # unicode keys are not valid for expansion into keyword arguments on method calls
     out_dict = {}
-    for key, value in in_dict.iteritems():
+    for key, value in iteritems(in_dict):
         out_dict[ str( key ) ] = value
     return out_dict
 
 
 def recursively_stringify_dictionary_keys( d ):
     if isinstance(d, dict):
-        return dict([(k.encode( DEFAULT_ENCODING ), recursively_stringify_dictionary_keys(v)) for k, v in d.iteritems()])
+        return dict([(k.encode( DEFAULT_ENCODING ), recursively_stringify_dictionary_keys(v)) for k, v in iteritems(d)])
     elif isinstance(d, list):
         return [recursively_stringify_dictionary_keys(x) for x in d]
     else:

lib/galaxy/util/heartbeat.py

@@ -4,6 +4,8 @@ import traceback
 import os
 import sys
 
+from six import iteritems
+
 
 def get_current_thread_object_dict():
     """
@@ -59,7 +61,7 @@ class Heartbeat( threading.Thread ):
                 print >> self.file
                 # Print the thread states
                 threads = get_current_thread_object_dict()
-                for thread_id, frame in sys._current_frames().iteritems():
+                for thread_id, frame in iteritems(sys._current_frames()):
                     if thread_id in threads:
                         object = repr( threads[thread_id] )
                     else:
@@ -141,7 +143,7 @@ class Heartbeat( threading.Thread ):
         print >> self.file_nonsleeping
         all_threads_are_sleeping = True
         threads = get_current_thread_object_dict()
-        for thread_id, frame in sys._current_frames().iteritems():
+        for thread_id, frame in iteritems(sys._current_frames()):
             if thread_id in threads:
                 object = repr( threads[thread_id] )
             else:

lib/galaxy/util/json.py

@@ -10,7 +10,7 @@ import math
 import random
 import string
 
-from six import text_type, string_types
+from six import text_type, string_types, iteritems
 
 dumps = json.dumps
 loads = json.loads
@@ -25,7 +25,7 @@ def json_fix( val ):
     if isinstance( val, list ):
         return [ json_fix( v ) for v in val ]
     elif isinstance( val, dict ):
-        return dict( [ ( json_fix( k ), json_fix( v ) ) for ( k, v ) in val.iteritems() ] )
+        return dict( [ ( json_fix( k ), json_fix( v ) ) for ( k, v ) in iteritems(val) ] )
     elif isinstance( val, text_type ):
         return val.encode( "utf8" )
     else:
@@ -42,7 +42,7 @@ def swap_inf_nan( val ):
     elif isinstance( val, collections.Sequence ):
         return [ swap_inf_nan( v ) for v in val ]
     elif isinstance( val, collections.Mapping ):
-        return dict( [ ( swap_inf_nan( k ), swap_inf_nan( v ) ) for ( k, v ) in val.iteritems() ] )
+        return dict( [ ( swap_inf_nan( k ), swap_inf_nan( v ) ) for ( k, v ) in iteritems(val) ] )
     elif isinstance(val, float):
         if math.isnan(val):
             return "__NaN__"

lib/galaxy/util/xml_macros.py

@@ -88,7 +88,7 @@ def _expand_tokens_for_el(element, tokens):
         new_value = _expand_tokens_str(element.text, tokens)
         if not (new_value is value):
             element.text = new_value
-    for key, value in element.attrib.iteritems():
+    for key, value in element.attrib.items():
         new_value = _expand_tokens_str(value, tokens)
         if not (new_value is value):
             element.attrib[key] = new_value
@@ -96,7 +96,7 @@ def _expand_tokens_for_el(element, tokens):
 
 
 def _expand_tokens_str(str, tokens):
-    for key, value in tokens.iteritems():
+    for key, value in tokens.items():
         if str.find(key) > -1:
             str = str.replace(key, value)
     return str