From 630ac3658f3c64bd9a9abe5757cce0ed01cad229 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 22 Dec 2015 13:43:01 +0000 Subject: [PATCH] Python 3 fixes and galaxy-lib updates from planemo. (Minus changes for conda which - should be merged as their own PR obviously.) --- lib/galaxy/exceptions/error_codes.py | 2 +- lib/galaxy/tools/deps/commands.py | 49 +++++++++++++++++-- lib/galaxy/tools/deps/containers.py | 4 +- lib/galaxy/tools/lint.py | 1 + lib/galaxy/tools/linters/inputs.py | 59 +++++++++++++++++------ lib/galaxy/tools/linters/stdio.py | 2 +- lib/galaxy/tools/parser/output_objects.py | 2 +- lib/galaxy/tools/parser/yaml.py | 6 +-- lib/galaxy/tools/toolbox/base.py | 3 +- lib/galaxy/tools/toolbox/panel.py | 3 +- lib/galaxy/util/__init__.py | 11 +++-- lib/galaxy/util/heartbeat.py | 6 ++- lib/galaxy/util/json.py | 6 +-- lib/galaxy/util/xml_macros.py | 4 +- 14 files changed, 119 insertions(+), 39 deletions(-) diff --git a/lib/galaxy/exceptions/error_codes.py b/lib/galaxy/exceptions/error_codes.py index 4afe67bea80..8b594efb250 100644 --- a/lib/galaxy/exceptions/error_codes.py +++ b/lib/galaxy/exceptions/error_codes.py @@ -26,7 +26,7 @@ class ErrorCode( object ): message = entry.get("message") return ( name, ErrorCode( code, message ) ) -error_codes_json = resource_string( __name__, 'error_codes.json' ) +error_codes_json = resource_string( __name__, 'error_codes.json' ).decode( "UTF-8" ) for entry in loads( error_codes_json ): name, error_code_obj = ErrorCode.from_dict( entry ) globals()[ name ] = error_code_obj diff --git a/lib/galaxy/tools/deps/commands.py b/lib/galaxy/tools/deps/commands.py index 5044d656247..64f15c6dc58 100644 --- a/lib/galaxy/tools/deps/commands.py +++ b/lib/galaxy/tools/deps/commands.py @@ -1,16 +1,49 @@ import os import subprocess +import sys as _sys -def shell(cmds, env=None): - p = shell_process(cmds, env) - return p.wait() +def redirecting_io(sys=_sys): + assert sys is not None + # We are redirecting standard out and standard error. + return not hasattr(sys.stdout, "fileno") + + +def redirect_aware_commmunicate(p, sys=_sys): + assert sys is not None + out, err = p.communicate() + if redirecting_io(sys=sys): + if out: + sys.stdout.write(out) + out = None + if err: + sys.stderr.write(err) + err = None + return out, err + + +def shell(cmds, env=None, **kwds): + sys = kwds.get("sys", _sys) + assert sys is not None + p = shell_process(cmds, env, **kwds) + if redirecting_io(sys=sys): + redirect_aware_commmunicate(p, sys=sys) + exit = p.returncode + return exit + else: + return p.wait() def shell_process(cmds, env=None, **kwds): + sys = kwds.get("sys", _sys) popen_kwds = dict( shell=True, ) + if kwds.get("stdout", None) is None and redirecting_io(sys=sys): + popen_kwds["stdout"] = subprocess.PIPE + if kwds.get("stderr", None) is None and redirecting_io(sys=sys): + popen_kwds["stderr"] = subprocess.PIPE + popen_kwds.update(**kwds) if env: new_env = os.environ.copy() @@ -41,6 +74,16 @@ def __wait(cmds, **popen_kwds): return stdout +def download_command(url, quote_url=False): + if quote_url: + url = "'%s'" % url + if which("wget"): + download_cmd = ["wget", "-q", "--recursive", "-O" "-", url] + else: + download_cmd = ["curl", "-L", url] + return download_cmd + + class CommandLineException(Exception): def __init__(self, command, stdout, stderr): diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py index 65334dd046c..35dfde5f38b 100644 --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -5,6 +5,8 @@ from abc import ( import os import string +import six + from galaxy.util import asbool from ..deps import docker_util @@ -208,7 +210,7 @@ class DockerContainer(Container): # Allow destinations to explicitly set environment variables just for # docker container. Better approach is to set for destination and then # pass through only what tool needs however. (See todo in ToolInfo.) - for key, value in self.destination_info.iteritems(): + for key, value in six.iteritems(self.destination_info): if key.startswith("docker_env_"): env = key[len("docker_env_"):] env_directives.append('"%s=%s"' % (env, value)) diff --git a/lib/galaxy/tools/lint.py b/lib/galaxy/tools/lint.py index 5d8400d6e75..fe699b511ce 100644 --- a/lib/galaxy/tools/lint.py +++ b/lib/galaxy/tools/lint.py @@ -10,6 +10,7 @@ LEVEL_ERROR = "error" def lint_xml(tool_xml, level=LEVEL_ALL, fail_level=LEVEL_WARN, extra_modules=[], skip_types=[]): lint_context = LintContext(level=level, skip_types=skip_types) lint_xml_with(lint_context, tool_xml, extra_modules) + return not lint_context.failed(fail_level) diff --git a/lib/galaxy/tools/linters/inputs.py b/lib/galaxy/tools/linters/inputs.py index 86d6682b716..85fe452c2ff 100644 --- a/lib/galaxy/tools/linters/inputs.py +++ b/lib/galaxy/tools/linters/inputs.py @@ -24,34 +24,49 @@ def lint_inputs(tool_xml, lint_ctx): if param_type == "data": if "format" not in param_attrib: lint_ctx.warn("Param input [%s] with no format specified - 'data' format will be assumed.", param_name) - # TODO: Validate type, much more... - conditional_selects = tool_xml.findall("./inputs//conditional") - for conditional in conditional_selects: - select = conditional.find('./param[@type="select"]') or [] - boolean = conditional.find('./param[@type="boolean"]') or [] - # Should conditionals ever not have a select? - if not len(select) and not len(boolean): - lint_ctx.warn("Conditional without or ") - continue + if param_type == "select": + dynamic_options = param.get("dynamic_options", None) + if dynamic_options is None: + dynamic_options = param.find("options") - if len(select): - select_options = select.findall('./option[@value]') + select_options = _find_with_attribute(param, 'option', 'value') if any(['value' not in option.attrib for option in select_options]): lint_ctx.error("Option without value") select_option_ids = [option.attrib.get('value', None) for option in select_options] - else: + + if dynamic_options is None and len(select_options) == 0: + message = "No options defined for select [%s]" % param_name + lint_ctx.warn(message) + + # TODO: Validate type, much more... + + conditional_selects = tool_xml.findall("./inputs//conditional") + for conditional in conditional_selects: + booleans = _find_with_attribute(conditional, "param", "type", "boolean") + selects = _find_with_attribute(conditional, "param", "type", "select") + # Should conditionals ever not have a select? + if not len(selects) and not len(booleans): + lint_ctx.warn("Conditional without or ") + continue + + for select in selects: + select_options = _find_with_attribute(select, 'option', 'value') + select_option_ids = [option.attrib.get('value', None) for option in select_options] + + for boolean in booleans: select_option_ids = [ - boolean.attrib.get('truevalue', 'True'), - boolean.attrib.get('falsevalue', 'False') + boolean.attrib.get('truevalue', 'true'), + boolean.attrib.get('falsevalue', 'false') ] whens = conditional.findall('./when') if any(['value' not in when.attrib for when in whens]): lint_ctx.error("When without value") - when_ids = [when.attrib.get('value', None) for when in whens] + when_ids = [w.attrib.get('value', None) for w in whens] + when_ids = [i.lower() if i in ["True", "False"] else i for i in when_ids] for select_id in select_option_ids: if select_id not in when_ids: @@ -82,3 +97,17 @@ def lint_repeats(tool_xml, lint_ctx): lint_ctx.error("Repeat does not specify name attribute.") if "title" not in repeat.attrib: lint_ctx.error("Repeat does not specify title attribute.") + + +def _find_with_attribute(element, tag, attribute, test_value=None): + rval = [] + for el in (element.findall('./%s' % tag) or []): + if attribute not in el.attrib: + continue + value = el.attrib[attribute] + if test_value is not None: + if value == test_value: + rval.append(el) + else: + rval.append(el) + return rval diff --git a/lib/galaxy/tools/linters/stdio.py b/lib/galaxy/tools/linters/stdio.py index de1358b565c..c18e14221ad 100644 --- a/lib/galaxy/tools/linters/stdio.py +++ b/lib/galaxy/tools/linters/stdio.py @@ -39,7 +39,7 @@ def _lint_exit_code(child, lint_ctx): def _lint_regex(child, lint_ctx): - for key, value in child.attrib.iteritems(): + for key, value in child.attrib.items(): if key == "source": if value not in ["stderr", "stdout", "both"]: lint_ctx.error("Unknown error code level encountered [%s]" % value) diff --git a/lib/galaxy/tools/parser/output_objects.py b/lib/galaxy/tools/parser/output_objects.py index 8ea45f081cf..b57d8fa3fa1 100644 --- a/lib/galaxy/tools/parser/output_objects.py +++ b/lib/galaxy/tools/parser/output_objects.py @@ -114,7 +114,7 @@ class ToolOutputCollection( ToolOutputBase ): # This line is probably not right - should verify structured_like # or have outputs and all outputs have name. if len( self.outputs ) > 1: - output_parts = [ToolOutputCollectionPart(self, k, v) for k, v in self.outputs.iteritems()] + output_parts = [ToolOutputCollectionPart(self, k, v) for k, v in self.outputs.items()] else: # either must have specified structured_like or something worse if self.structure.structured_like: diff --git a/lib/galaxy/tools/parser/yaml.py b/lib/galaxy/tools/parser/yaml.py index 571ad881dcc..0455d619d99 100644 --- a/lib/galaxy/tools/parser/yaml.py +++ b/lib/galaxy/tools/parser/yaml.py @@ -169,7 +169,7 @@ def _parse_test(i, test_dict): inputs = test_dict["inputs"] if _is_dict(inputs): new_inputs = [] - for key, value in inputs.iteritems(): + for key, value in inputs.items(): new_inputs.append((key, value, {})) test_dict["inputs"] = new_inputs @@ -177,7 +177,7 @@ def _parse_test(i, test_dict): new_outputs = [] if _is_dict(outputs): - for key, value in outputs.iteritems(): + for key, value in outputs.items(): if _is_dict(value): attributes = value file = attributes.get("file") @@ -322,6 +322,6 @@ class YamlInputSource(InputSource): def _ensure_has(dict, defaults): - for key, value in defaults.iteritems(): + for key, value in defaults.items(): if key not in dict: dict[key] = value diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index d1c230da1fa..49ba91ef773 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -7,6 +7,7 @@ import tempfile from markupsafe import escape from six.moves.urllib.parse import urlparse +from six import iteritems from galaxy.util.dictifiable import Dictifiable @@ -456,7 +457,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): return [] def tools( self ): - return self._tools_by_id.iteritems() + return iteritems(self._tools_by_id) def dynamic_confs( self, include_migrated_tool_conf=False ): confs = [] diff --git a/lib/galaxy/tools/toolbox/panel.py b/lib/galaxy/tools/toolbox/panel.py index 1f6ec7efbbb..9e77231aceb 100644 --- a/lib/galaxy/tools/toolbox/panel.py +++ b/lib/galaxy/tools/toolbox/panel.py @@ -4,6 +4,7 @@ from galaxy.util.odict import odict from galaxy.util import bunch from galaxy.util.dictifiable import Dictifiable +from six import iteritems panel_item_types = bunch.Bunch( TOOL="TOOL", @@ -27,7 +28,7 @@ class HasPanelItems: """ Iterate through panel items each represented as a tuple of (panel_key, panel_type, panel_content). """ - for panel_key, panel_value in self.panel_items().iteritems(): + for panel_key, panel_value in iteritems(self.panel_items()): if panel_value is None: continue panel_type = panel_item_types.SECTION diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 1e1f2c9b549..542d11a3310 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -23,6 +23,7 @@ import time import tempfile import threading from six.moves.urllib import parse as urlparse +from six import iteritems from galaxy.util import json from datetime import datetime @@ -226,17 +227,17 @@ def xml_element_to_dict( elem ): if sub_elems: sub_elem_dict = dict() for sub_sub_elem_dict in map( xml_element_to_dict, sub_elems ): - for key, value in sub_sub_elem_dict.iteritems(): + for key, value in iteritems(sub_sub_elem_dict): if key not in sub_elem_dict: sub_elem_dict[ key ] = [] sub_elem_dict[ key ].append( value ) - for key, value in sub_elem_dict.iteritems(): + for key, value in iteritems(sub_elem_dict): if len( value ) == 1: rval[ elem.tag ][ key ] = value[0] else: rval[ elem.tag ][ key ] = value if elem.attrib: - for key, value in elem.attrib.iteritems(): + for key, value in iteritems(elem.attrib): rval[ elem.tag ][ "@%s" % key ] = value if elem.text: @@ -1010,14 +1011,14 @@ def stringify_dictionary_keys( in_dict ): # changes unicode keys into strings, only works on top level (does not recurse) # unicode keys are not valid for expansion into keyword arguments on method calls out_dict = {} - for key, value in in_dict.iteritems(): + for key, value in iteritems(in_dict): out_dict[ str( key ) ] = value return out_dict def recursively_stringify_dictionary_keys( d ): if isinstance(d, dict): - return dict([(k.encode( DEFAULT_ENCODING ), recursively_stringify_dictionary_keys(v)) for k, v in d.iteritems()]) + return dict([(k.encode( DEFAULT_ENCODING ), recursively_stringify_dictionary_keys(v)) for k, v in iteritems(d)]) elif isinstance(d, list): return [recursively_stringify_dictionary_keys(x) for x in d] else: diff --git a/lib/galaxy/util/heartbeat.py b/lib/galaxy/util/heartbeat.py index 59dc7a12031..d9be8104e54 100644 --- a/lib/galaxy/util/heartbeat.py +++ b/lib/galaxy/util/heartbeat.py @@ -4,6 +4,8 @@ import traceback import os import sys +from six import iteritems + def get_current_thread_object_dict(): """ @@ -59,7 +61,7 @@ class Heartbeat( threading.Thread ): print >> self.file # Print the thread states threads = get_current_thread_object_dict() - for thread_id, frame in sys._current_frames().iteritems(): + for thread_id, frame in iteritems(sys._current_frames()): if thread_id in threads: object = repr( threads[thread_id] ) else: @@ -141,7 +143,7 @@ class Heartbeat( threading.Thread ): print >> self.file_nonsleeping all_threads_are_sleeping = True threads = get_current_thread_object_dict() - for thread_id, frame in sys._current_frames().iteritems(): + for thread_id, frame in iteritems(sys._current_frames()): if thread_id in threads: object = repr( threads[thread_id] ) else: diff --git a/lib/galaxy/util/json.py b/lib/galaxy/util/json.py index 1f9b5b44c4e..dc9a16289a5 100644 --- a/lib/galaxy/util/json.py +++ b/lib/galaxy/util/json.py @@ -10,7 +10,7 @@ import math import random import string -from six import text_type, string_types +from six import text_type, string_types, iteritems dumps = json.dumps loads = json.loads @@ -25,7 +25,7 @@ def json_fix( val ): if isinstance( val, list ): return [ json_fix( v ) for v in val ] elif isinstance( val, dict ): - return dict( [ ( json_fix( k ), json_fix( v ) ) for ( k, v ) in val.iteritems() ] ) + return dict( [ ( json_fix( k ), json_fix( v ) ) for ( k, v ) in iteritems(val) ] ) elif isinstance( val, text_type ): return val.encode( "utf8" ) else: @@ -42,7 +42,7 @@ def swap_inf_nan( val ): elif isinstance( val, collections.Sequence ): return [ swap_inf_nan( v ) for v in val ] elif isinstance( val, collections.Mapping ): - return dict( [ ( swap_inf_nan( k ), swap_inf_nan( v ) ) for ( k, v ) in val.iteritems() ] ) + return dict( [ ( swap_inf_nan( k ), swap_inf_nan( v ) ) for ( k, v ) in iteritems(val) ] ) elif isinstance(val, float): if math.isnan(val): return "__NaN__" diff --git a/lib/galaxy/util/xml_macros.py b/lib/galaxy/util/xml_macros.py index 8a03c5e58d9..3ebc0d04912 100644 --- a/lib/galaxy/util/xml_macros.py +++ b/lib/galaxy/util/xml_macros.py @@ -88,7 +88,7 @@ def _expand_tokens_for_el(element, tokens): new_value = _expand_tokens_str(element.text, tokens) if not (new_value is value): element.text = new_value - for key, value in element.attrib.iteritems(): + for key, value in element.attrib.items(): new_value = _expand_tokens_str(value, tokens) if not (new_value is value): element.attrib[key] = new_value @@ -96,7 +96,7 @@ def _expand_tokens_for_el(element, tokens): def _expand_tokens_str(str, tokens): - for key, value in tokens.iteritems(): + for key, value in tokens.items(): if str.find(key) > -1: str = str.replace(key, value) return str