diff --git a/lib/galaxy/exceptions/error_codes.py b/lib/galaxy/exceptions/error_codes.py
index 4afe67bea80..8b594efb250 100644
--- a/lib/galaxy/exceptions/error_codes.py
+++ b/lib/galaxy/exceptions/error_codes.py
@@ -26,7 +26,7 @@ class ErrorCode( object ):
message = entry.get("message")
return ( name, ErrorCode( code, message ) )
-error_codes_json = resource_string( __name__, 'error_codes.json' )
+error_codes_json = resource_string( __name__, 'error_codes.json' ).decode( "UTF-8" )
for entry in loads( error_codes_json ):
name, error_code_obj = ErrorCode.from_dict( entry )
globals()[ name ] = error_code_obj
diff --git a/lib/galaxy/tools/deps/commands.py b/lib/galaxy/tools/deps/commands.py
index 5044d656247..64f15c6dc58 100644
--- a/lib/galaxy/tools/deps/commands.py
+++ b/lib/galaxy/tools/deps/commands.py
@@ -1,16 +1,49 @@
import os
import subprocess
+import sys as _sys
-def shell(cmds, env=None):
- p = shell_process(cmds, env)
- return p.wait()
+def redirecting_io(sys=_sys):
+ assert sys is not None
+ # We are redirecting standard out and standard error.
+ return not hasattr(sys.stdout, "fileno")
+
+
+def redirect_aware_commmunicate(p, sys=_sys):
+ assert sys is not None
+ out, err = p.communicate()
+ if redirecting_io(sys=sys):
+ if out:
+ sys.stdout.write(out)
+ out = None
+ if err:
+ sys.stderr.write(err)
+ err = None
+ return out, err
+
+
+def shell(cmds, env=None, **kwds):
+ sys = kwds.get("sys", _sys)
+ assert sys is not None
+ p = shell_process(cmds, env, **kwds)
+ if redirecting_io(sys=sys):
+ redirect_aware_commmunicate(p, sys=sys)
+ exit = p.returncode
+ return exit
+ else:
+ return p.wait()
def shell_process(cmds, env=None, **kwds):
+ sys = kwds.get("sys", _sys)
popen_kwds = dict(
shell=True,
)
+ if kwds.get("stdout", None) is None and redirecting_io(sys=sys):
+ popen_kwds["stdout"] = subprocess.PIPE
+ if kwds.get("stderr", None) is None and redirecting_io(sys=sys):
+ popen_kwds["stderr"] = subprocess.PIPE
+
popen_kwds.update(**kwds)
if env:
new_env = os.environ.copy()
@@ -41,6 +74,16 @@ def __wait(cmds, **popen_kwds):
return stdout
+def download_command(url, quote_url=False):
+ if quote_url:
+ url = "'%s'" % url
+ if which("wget"):
+ download_cmd = ["wget", "-q", "--recursive", "-O" "-", url]
+ else:
+ download_cmd = ["curl", "-L", url]
+ return download_cmd
+
+
class CommandLineException(Exception):
def __init__(self, command, stdout, stderr):
diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py
index 65334dd046c..35dfde5f38b 100644
--- a/lib/galaxy/tools/deps/containers.py
+++ b/lib/galaxy/tools/deps/containers.py
@@ -5,6 +5,8 @@ from abc import (
import os
import string
+import six
+
from galaxy.util import asbool
from ..deps import docker_util
@@ -208,7 +210,7 @@ class DockerContainer(Container):
# Allow destinations to explicitly set environment variables just for
# docker container. Better approach is to set for destination and then
# pass through only what tool needs however. (See todo in ToolInfo.)
- for key, value in self.destination_info.iteritems():
+ for key, value in six.iteritems(self.destination_info):
if key.startswith("docker_env_"):
env = key[len("docker_env_"):]
env_directives.append('"%s=%s"' % (env, value))
diff --git a/lib/galaxy/tools/lint.py b/lib/galaxy/tools/lint.py
index 5d8400d6e75..fe699b511ce 100644
--- a/lib/galaxy/tools/lint.py
+++ b/lib/galaxy/tools/lint.py
@@ -10,6 +10,7 @@ LEVEL_ERROR = "error"
def lint_xml(tool_xml, level=LEVEL_ALL, fail_level=LEVEL_WARN, extra_modules=[], skip_types=[]):
lint_context = LintContext(level=level, skip_types=skip_types)
lint_xml_with(lint_context, tool_xml, extra_modules)
+
return not lint_context.failed(fail_level)
diff --git a/lib/galaxy/tools/linters/inputs.py b/lib/galaxy/tools/linters/inputs.py
index 86d6682b716..85fe452c2ff 100644
--- a/lib/galaxy/tools/linters/inputs.py
+++ b/lib/galaxy/tools/linters/inputs.py
@@ -24,34 +24,49 @@ def lint_inputs(tool_xml, lint_ctx):
if param_type == "data":
if "format" not in param_attrib:
lint_ctx.warn("Param input [%s] with no format specified - 'data' format will be assumed.", param_name)
- # TODO: Validate type, much more...
- conditional_selects = tool_xml.findall("./inputs//conditional")
- for conditional in conditional_selects:
- select = conditional.find('./param[@type="select"]') or []
- boolean = conditional.find('./param[@type="boolean"]') or []
- # Should conditionals ever not have a select?
- if not len(select) and not len(boolean):
- lint_ctx.warn("Conditional without or ")
- continue
+ if param_type == "select":
+ dynamic_options = param.get("dynamic_options", None)
+ if dynamic_options is None:
+ dynamic_options = param.find("options")
- if len(select):
- select_options = select.findall('./option[@value]')
+ select_options = _find_with_attribute(param, 'option', 'value')
if any(['value' not in option.attrib for option in select_options]):
lint_ctx.error("Option without value")
select_option_ids = [option.attrib.get('value', None) for option in select_options]
- else:
+
+ if dynamic_options is None and len(select_options) == 0:
+ message = "No options defined for select [%s]" % param_name
+ lint_ctx.warn(message)
+
+ # TODO: Validate type, much more...
+
+ conditional_selects = tool_xml.findall("./inputs//conditional")
+ for conditional in conditional_selects:
+ booleans = _find_with_attribute(conditional, "param", "type", "boolean")
+ selects = _find_with_attribute(conditional, "param", "type", "select")
+ # Should conditionals ever not have a select?
+ if not len(selects) and not len(booleans):
+ lint_ctx.warn("Conditional without or ")
+ continue
+
+ for select in selects:
+ select_options = _find_with_attribute(select, 'option', 'value')
+ select_option_ids = [option.attrib.get('value', None) for option in select_options]
+
+ for boolean in booleans:
select_option_ids = [
- boolean.attrib.get('truevalue', 'True'),
- boolean.attrib.get('falsevalue', 'False')
+ boolean.attrib.get('truevalue', 'true'),
+ boolean.attrib.get('falsevalue', 'false')
]
whens = conditional.findall('./when')
if any(['value' not in when.attrib for when in whens]):
lint_ctx.error("When without value")
- when_ids = [when.attrib.get('value', None) for when in whens]
+ when_ids = [w.attrib.get('value', None) for w in whens]
+ when_ids = [i.lower() if i in ["True", "False"] else i for i in when_ids]
for select_id in select_option_ids:
if select_id not in when_ids:
@@ -82,3 +97,17 @@ def lint_repeats(tool_xml, lint_ctx):
lint_ctx.error("Repeat does not specify name attribute.")
if "title" not in repeat.attrib:
lint_ctx.error("Repeat does not specify title attribute.")
+
+
+def _find_with_attribute(element, tag, attribute, test_value=None):
+ rval = []
+ for el in (element.findall('./%s' % tag) or []):
+ if attribute not in el.attrib:
+ continue
+ value = el.attrib[attribute]
+ if test_value is not None:
+ if value == test_value:
+ rval.append(el)
+ else:
+ rval.append(el)
+ return rval
diff --git a/lib/galaxy/tools/linters/stdio.py b/lib/galaxy/tools/linters/stdio.py
index de1358b565c..c18e14221ad 100644
--- a/lib/galaxy/tools/linters/stdio.py
+++ b/lib/galaxy/tools/linters/stdio.py
@@ -39,7 +39,7 @@ def _lint_exit_code(child, lint_ctx):
def _lint_regex(child, lint_ctx):
- for key, value in child.attrib.iteritems():
+ for key, value in child.attrib.items():
if key == "source":
if value not in ["stderr", "stdout", "both"]:
lint_ctx.error("Unknown error code level encountered [%s]" % value)
diff --git a/lib/galaxy/tools/parser/output_objects.py b/lib/galaxy/tools/parser/output_objects.py
index 8ea45f081cf..b57d8fa3fa1 100644
--- a/lib/galaxy/tools/parser/output_objects.py
+++ b/lib/galaxy/tools/parser/output_objects.py
@@ -114,7 +114,7 @@ class ToolOutputCollection( ToolOutputBase ):
# This line is probably not right - should verify structured_like
# or have outputs and all outputs have name.
if len( self.outputs ) > 1:
- output_parts = [ToolOutputCollectionPart(self, k, v) for k, v in self.outputs.iteritems()]
+ output_parts = [ToolOutputCollectionPart(self, k, v) for k, v in self.outputs.items()]
else:
# either must have specified structured_like or something worse
if self.structure.structured_like:
diff --git a/lib/galaxy/tools/parser/yaml.py b/lib/galaxy/tools/parser/yaml.py
index 571ad881dcc..0455d619d99 100644
--- a/lib/galaxy/tools/parser/yaml.py
+++ b/lib/galaxy/tools/parser/yaml.py
@@ -169,7 +169,7 @@ def _parse_test(i, test_dict):
inputs = test_dict["inputs"]
if _is_dict(inputs):
new_inputs = []
- for key, value in inputs.iteritems():
+ for key, value in inputs.items():
new_inputs.append((key, value, {}))
test_dict["inputs"] = new_inputs
@@ -177,7 +177,7 @@ def _parse_test(i, test_dict):
new_outputs = []
if _is_dict(outputs):
- for key, value in outputs.iteritems():
+ for key, value in outputs.items():
if _is_dict(value):
attributes = value
file = attributes.get("file")
@@ -322,6 +322,6 @@ class YamlInputSource(InputSource):
def _ensure_has(dict, defaults):
- for key, value in defaults.iteritems():
+ for key, value in defaults.items():
if key not in dict:
dict[key] = value
diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py
index d1c230da1fa..49ba91ef773 100644
--- a/lib/galaxy/tools/toolbox/base.py
+++ b/lib/galaxy/tools/toolbox/base.py
@@ -7,6 +7,7 @@ import tempfile
from markupsafe import escape
from six.moves.urllib.parse import urlparse
+from six import iteritems
from galaxy.util.dictifiable import Dictifiable
@@ -456,7 +457,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
return []
def tools( self ):
- return self._tools_by_id.iteritems()
+ return iteritems(self._tools_by_id)
def dynamic_confs( self, include_migrated_tool_conf=False ):
confs = []
diff --git a/lib/galaxy/tools/toolbox/panel.py b/lib/galaxy/tools/toolbox/panel.py
index 1f6ec7efbbb..9e77231aceb 100644
--- a/lib/galaxy/tools/toolbox/panel.py
+++ b/lib/galaxy/tools/toolbox/panel.py
@@ -4,6 +4,7 @@ from galaxy.util.odict import odict
from galaxy.util import bunch
from galaxy.util.dictifiable import Dictifiable
+from six import iteritems
panel_item_types = bunch.Bunch(
TOOL="TOOL",
@@ -27,7 +28,7 @@ class HasPanelItems:
""" Iterate through panel items each represented as a tuple of
(panel_key, panel_type, panel_content).
"""
- for panel_key, panel_value in self.panel_items().iteritems():
+ for panel_key, panel_value in iteritems(self.panel_items()):
if panel_value is None:
continue
panel_type = panel_item_types.SECTION
diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py
index 1e1f2c9b549..542d11a3310 100644
--- a/lib/galaxy/util/__init__.py
+++ b/lib/galaxy/util/__init__.py
@@ -23,6 +23,7 @@ import time
import tempfile
import threading
from six.moves.urllib import parse as urlparse
+from six import iteritems
from galaxy.util import json
from datetime import datetime
@@ -226,17 +227,17 @@ def xml_element_to_dict( elem ):
if sub_elems:
sub_elem_dict = dict()
for sub_sub_elem_dict in map( xml_element_to_dict, sub_elems ):
- for key, value in sub_sub_elem_dict.iteritems():
+ for key, value in iteritems(sub_sub_elem_dict):
if key not in sub_elem_dict:
sub_elem_dict[ key ] = []
sub_elem_dict[ key ].append( value )
- for key, value in sub_elem_dict.iteritems():
+ for key, value in iteritems(sub_elem_dict):
if len( value ) == 1:
rval[ elem.tag ][ key ] = value[0]
else:
rval[ elem.tag ][ key ] = value
if elem.attrib:
- for key, value in elem.attrib.iteritems():
+ for key, value in iteritems(elem.attrib):
rval[ elem.tag ][ "@%s" % key ] = value
if elem.text:
@@ -1010,14 +1011,14 @@ def stringify_dictionary_keys( in_dict ):
# changes unicode keys into strings, only works on top level (does not recurse)
# unicode keys are not valid for expansion into keyword arguments on method calls
out_dict = {}
- for key, value in in_dict.iteritems():
+ for key, value in iteritems(in_dict):
out_dict[ str( key ) ] = value
return out_dict
def recursively_stringify_dictionary_keys( d ):
if isinstance(d, dict):
- return dict([(k.encode( DEFAULT_ENCODING ), recursively_stringify_dictionary_keys(v)) for k, v in d.iteritems()])
+ return dict([(k.encode( DEFAULT_ENCODING ), recursively_stringify_dictionary_keys(v)) for k, v in iteritems(d)])
elif isinstance(d, list):
return [recursively_stringify_dictionary_keys(x) for x in d]
else:
diff --git a/lib/galaxy/util/heartbeat.py b/lib/galaxy/util/heartbeat.py
index 59dc7a12031..d9be8104e54 100644
--- a/lib/galaxy/util/heartbeat.py
+++ b/lib/galaxy/util/heartbeat.py
@@ -4,6 +4,8 @@ import traceback
import os
import sys
+from six import iteritems
+
def get_current_thread_object_dict():
"""
@@ -59,7 +61,7 @@ class Heartbeat( threading.Thread ):
print >> self.file
# Print the thread states
threads = get_current_thread_object_dict()
- for thread_id, frame in sys._current_frames().iteritems():
+ for thread_id, frame in iteritems(sys._current_frames()):
if thread_id in threads:
object = repr( threads[thread_id] )
else:
@@ -141,7 +143,7 @@ class Heartbeat( threading.Thread ):
print >> self.file_nonsleeping
all_threads_are_sleeping = True
threads = get_current_thread_object_dict()
- for thread_id, frame in sys._current_frames().iteritems():
+ for thread_id, frame in iteritems(sys._current_frames()):
if thread_id in threads:
object = repr( threads[thread_id] )
else:
diff --git a/lib/galaxy/util/json.py b/lib/galaxy/util/json.py
index 1f9b5b44c4e..dc9a16289a5 100644
--- a/lib/galaxy/util/json.py
+++ b/lib/galaxy/util/json.py
@@ -10,7 +10,7 @@ import math
import random
import string
-from six import text_type, string_types
+from six import text_type, string_types, iteritems
dumps = json.dumps
loads = json.loads
@@ -25,7 +25,7 @@ def json_fix( val ):
if isinstance( val, list ):
return [ json_fix( v ) for v in val ]
elif isinstance( val, dict ):
- return dict( [ ( json_fix( k ), json_fix( v ) ) for ( k, v ) in val.iteritems() ] )
+ return dict( [ ( json_fix( k ), json_fix( v ) ) for ( k, v ) in iteritems(val) ] )
elif isinstance( val, text_type ):
return val.encode( "utf8" )
else:
@@ -42,7 +42,7 @@ def swap_inf_nan( val ):
elif isinstance( val, collections.Sequence ):
return [ swap_inf_nan( v ) for v in val ]
elif isinstance( val, collections.Mapping ):
- return dict( [ ( swap_inf_nan( k ), swap_inf_nan( v ) ) for ( k, v ) in val.iteritems() ] )
+ return dict( [ ( swap_inf_nan( k ), swap_inf_nan( v ) ) for ( k, v ) in iteritems(val) ] )
elif isinstance(val, float):
if math.isnan(val):
return "__NaN__"
diff --git a/lib/galaxy/util/xml_macros.py b/lib/galaxy/util/xml_macros.py
index 8a03c5e58d9..3ebc0d04912 100644
--- a/lib/galaxy/util/xml_macros.py
+++ b/lib/galaxy/util/xml_macros.py
@@ -88,7 +88,7 @@ def _expand_tokens_for_el(element, tokens):
new_value = _expand_tokens_str(element.text, tokens)
if not (new_value is value):
element.text = new_value
- for key, value in element.attrib.iteritems():
+ for key, value in element.attrib.items():
new_value = _expand_tokens_str(value, tokens)
if not (new_value is value):
element.attrib[key] = new_value
@@ -96,7 +96,7 @@ def _expand_tokens_for_el(element, tokens):
def _expand_tokens_str(str, tokens):
- for key, value in tokens.iteritems():
+ for key, value in tokens.items():
if str.find(key) > -1:
str = str.replace(key, value)
return str