Merge branch 'feature/plugin-system' of github.com:bgruening/galaxy into feature/plugin-system

.ci/flake8_lint_include_list.txt

@@ -95,6 +95,7 @@ lib/galaxy/main.py
 lib/galaxy/managers/annotatable.py
 lib/galaxy/managers/api_keys.py
 lib/galaxy/managers/base.py
+lib/galaxy/managers/collections_util.py
 lib/galaxy/managers/context.py
 lib/galaxy/managers/deletable.py
 lib/galaxy/managers/__init__.py
@@ -103,7 +104,9 @@ lib/galaxy/managers/libraries.py
 lib/galaxy/managers/secured.py
 lib/galaxy/managers/taggable.py
 lib/galaxy/model/__init__.py
+lib/galaxy/model/item_attrs.py
 lib/galaxy/model/mapping.py
+lib/galaxy/model/metadata.py
 lib/galaxy/model/migrate/__init__.py
 lib/galaxy/model/migrate/versions/0001_initial_tables.py
 lib/galaxy/model/migrate/versions/0002_metadata_file_table.py
@@ -246,9 +249,11 @@ lib/galaxy/sample_tracking/sample.py
 lib/galaxy/security/validate_user_input.py
 lib/galaxy/tags/
 lib/galaxy/tools/actions/metadata.py
-lib/galaxy/tools/cwl
+lib/galaxy/tools/cwl/
 lib/galaxy/tools/data_manager/__init__.py
-lib/galaxy/tools/deps
+lib/galaxy/tools/deps/
+lib/galaxy/tools/exception_handling.py
+lib/galaxy/tools/execute.py
 lib/galaxy/tools/filters/
 lib/galaxy/tools/imp_exp/export_history.py
 lib/galaxy/tools/imp_exp/__init__.py
@@ -256,11 +261,15 @@ lib/galaxy/tools/linters/
 lib/galaxy/tools/lint.py
 lib/galaxy/tools/lint_util.py
 lib/galaxy/tools/loader_directory.py
+lib/galaxy/tools/loader.py
+lib/galaxy/tools/parameters/dataset_matcher.py
 lib/galaxy/tools/parameters/history_query.py
+lib/galaxy/tools/parameters/__init__.py
 lib/galaxy/tools/parameters/input_translation.py
 lib/galaxy/tools/parameters/sanitize.py
 lib/galaxy/tools/parameters/validation.py
 lib/galaxy/tools/parameters/wrapped_json.py
+lib/galaxy/tools/parameters/wrapped.py
 lib/galaxy/tools/parser/
 lib/galaxy/tools/special_tools.py
 lib/galaxy/tools/test.py
@@ -347,9 +356,11 @@ lib/galaxy/web/framework/middleware/statsd.py
 lib/galaxy/web/framework/middleware/translogger.py
 lib/galaxy/web/framework/middleware/xforwardedhost.py
 lib/galaxy/web/params.py
+lib/galaxy/workflow/extract.py
 lib/galaxy/workflow/__init__.py
 lib/galaxy/workflow/render.py
-lib/galaxy/workflow/schedulers/__init__.py
+lib/galaxy/workflow/run.py
+lib/galaxy/workflow/schedulers/
 lib/galaxy/workflow/steps.py
 lib/galaxy/work/__init__.py
 lib/mimeparse.py
@@ -517,6 +528,7 @@ test/unit/tools/test_tool_deps.py
 test/unit/tools/test_tool_loader.py
 test/unit/tools/test_toolbox_filters.py
 test/unit/tools/test_watcher.py
+test/unit/tools_support.py
 test/unit/unittest_utils/__init__.py
 test/unit/unittest_utils/utility.py
 test/unit/visualizations/__init__.py

.ci/py3_sources.txt

@@ -28,8 +28,11 @@ lib/galaxy/forms/
 lib/galaxy/jobs/
 lib/galaxy/managers/
 lib/galaxy/model/__init__.py
+lib/galaxy/model/item_attrs.py
 lib/galaxy/model/mapping.py
+lib/galaxy/model/metadata.py
 lib/galaxy/model/migrate/
+lib/galaxy/model/orm/now.py
 lib/galaxy/objectstore/
 lib/galaxy/openid/
 lib/galaxy/quota/
@@ -38,17 +41,26 @@ lib/galaxy/security/
 lib/galaxy/tags/
 lib/galaxy/tools/cwl/
 lib/galaxy/tools/deps/
+lib/galaxy/tools/exception_handling.py
+lib/galaxy/tools/execute.py
 lib/galaxy/tools/lint.py
 lib/galaxy/tools/lint_util.py
 lib/galaxy/tools/linters/
 lib/galaxy/tools/loader.py
 lib/galaxy/tools/loader_directory.py
+lib/galaxy/tools/parameters/dataset_matcher.py
+lib/galaxy/tools/parameters/__init__.py
+lib/galaxy/tools/parameters/wrapped_json.py
+lib/galaxy/tools/parameters/wrapped.py
 lib/galaxy/tools/parser/
 lib/galaxy/tools/toolbox/
 lib/galaxy/tours/
 lib/galaxy/util/
 lib/galaxy/visualization/
 lib/galaxy/work/
+lib/galaxy/workflow/extract.py
+lib/galaxy/workflow/run.py
+lib/galaxy/workflow/schedulers/core.py
 lib/galaxy_ext/
 lib/galaxy_utils/
 lib/log_tempfile.py

config/galaxy.ini.sample

@@ -890,8 +890,7 @@ use_interactive = True
 # Galaxy encodes various internal values when these values will be output in
 # some format (for example, in a URL or cookie).  You should set a key to be
 # used by the algorithm that encodes and decodes these values.  It can be any
-# string.  If left unchanged, anyone could construct a cookie that would grant
-# them access to others' sessions.
+# string.
 # One simple way to generate a value for this is with the shell command:
 #   python -c 'import time; print time.time()' | md5sum | cut -f 1 -d ' '
 #id_secret = USING THE DEFAULT IS NOT SECURE!

config/plugins/webhooks/demo/trans_object/helper/__init__.py

@@ -1,2 +1,6 @@
 def main(trans, webhook):
-    return {'username': trans.user.username}
+    if trans.user:
+        user = trans.user.username
+    else:
+        user = 'No user is logged in.'
+    return {'username': user}

doc/source/admin/webhooks.rst

@@ -7,17 +7,11 @@ can consists out of simple HTML, JS or dynamically generated content from a pyth
 
 Plugin activation
 -----------------
-Webhooks are deactivated by default.
-To activate webhooks make sure the path in your `galaxy.ini` it set and that you activate all plugins that you want to use manually by changing the config of each webhook:
-
-.. code-block:: none
-
-  config/plugins/webhooks/{TOOL_NAME}/config/{TOOL_NAME}.yml
-and set the parameter activate to true:
-
-.. code-block:: python
-
-  activate: true 
+All webhooks that are included in the main Galaxy distribution are located in the `config/plugins/webhooks/demo` folder
+and are deactivated by default.
+To activate these demo webhooks make sure this path is added to `webhooks_dir` in your `galaxy.ini`. You can add as many 
+webhook folders as you like as a comma separated list. 
+Webhooks supports one additionl layer of activating/deactivating by changing the `activate: true` in each config of each webhook.
   
 
 Entry points

lib/galaxy/managers/collections_util.py

@@ -1,8 +1,7 @@
-from galaxy import exceptions
-from galaxy import web
-from galaxy import model
-
 import logging
+
+from galaxy import exceptions, model, web
+
 log = logging.getLogger( __name__ )
 
 ERROR_MESSAGE_UNKNOWN_SRC = "Unknown dataset source (src) %s."
@@ -77,7 +76,7 @@ def dictify_dataset_collection_instance( dataset_collection_instance, parent, se
         dict_value[ 'url' ] = web.url_for( 'library_content', library_id=encoded_library_id, id=encoded_id, folder_id=encoded_folder_id )
     if view == "element":
         collection = dataset_collection_instance.collection
-        dict_value[ 'elements' ] = map( dictify_element, collection.elements )
+        dict_value[ 'elements' ] = [ dictify_element(_) for _ in collection.elements ]
         dict_value[ 'populated' ] = collection.populated
     security.encode_all_ids( dict_value, recursive=True )  # TODO: Use Kyle's recursive formulation of this.
     return dict_value
@@ -89,10 +88,10 @@ def dictify_element( element ):
     if element.child_collection:
         # Recursively yield elements for each nested collection...
         child_collection = element.child_collection
-        object_detials[ "elements" ] = map( dictify_element, child_collection.elements )
+        object_detials[ "elements" ] = [ dictify_element(_) for _ in child_collection.elements ]
         object_detials[ "populated" ] = child_collection.populated
 
     dictified[ "object" ] = object_detials
     return dictified
 
-__all__ = [ api_payload_to_create_params, dictify_dataset_collection_instance ]
+__all__ = ( 'api_payload_to_create_params', 'dictify_dataset_collection_instance' )

lib/galaxy/model/item_attrs.py

@@ -1,7 +1,9 @@
+import logging
+
 from sqlalchemy.sql.expression import func
+
 # Cannot import galaxy.model b/c it creates a circular import graph.
 import galaxy
-import logging
 
 log = logging.getLogger( __name__ )
 
@@ -162,8 +164,8 @@ class UsesAnnotations:
         return getattr( galaxy.model, class_name, None )
 
 
-__all__ = [
+__all__ = (
     'UsesAnnotations',
     'UsesItemRatings',
     'RuntimeException',
-]
+)

lib/galaxy/model/metadata.py

@@ -4,7 +4,6 @@ Galaxy Metadata
 """
 
 import copy
-import cPickle
 import json
 import logging
 import os
@@ -15,6 +14,7 @@ import weakref
 from os.path import abspath
 
 from six import string_types
+from six.moves import cPickle
 from sqlalchemy.orm import object_session
 
 import galaxy.model
@@ -93,13 +93,14 @@ class MetadataCollection( object ):
             return default
 
     def items(self):
-        return iter( [ ( k, self.get( k ) ) for k in self.spec.iterkeys() ] )
+        return iter( [ ( k, self.get( k ) ) for k in self.spec.keys() ] )
 
     def __str__(self):
         return dict( self.items() ).__str__()
 
-    def __nonzero__( self ):
+    def __bool__( self ):
         return bool( self.parent._metadata )
+    __nonzero__ = __bool__
 
     def __getattr__( self, name ):
         if name in self.spec:
@@ -205,7 +206,7 @@ class MetadataSpecCollection( odict ):
         self[item.name] = item
 
     def iter( self ):
-        return self.itervalues()
+        return iter(self.values())
 
     def __getattr__( self, name ):
         return self.get( name )
@@ -452,7 +453,7 @@ class RangeParameter( SelectParameter ):
         other_values = other_values or {}
 
         if values is None:
-            values = zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step ))
+            values = list(zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step ) ))
         return SelectParameter.get_html_field( self, value=value, context=context, other_values=other_values, values=values, **kwd )
 
     def get_html( self, value, context=None, other_values=None, values=None, **kwd ):
@@ -460,7 +461,7 @@ class RangeParameter( SelectParameter ):
         other_values = other_values or {}
 
         if values is None:
-            values = zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step ))
+            values = list(zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step ) ))
         return SelectParameter.get_html( self, value, context=context, other_values=other_values, values=values, **kwd )
 
     @classmethod
@@ -478,7 +479,7 @@ class ColumnParameter( RangeParameter ):
 
         if values is None and context:
             column_range = range( 1, ( context.columns or 0 ) + 1, 1 )
-            values = zip( column_range, column_range )
+            values = list(zip( column_range, column_range ))
         return RangeParameter.get_html_field( self, value=value, context=context, other_values=other_values, values=values, **kwd )
 
     def get_html( self, value, context=None, other_values=None, values=None, **kwd ):
@@ -487,7 +488,7 @@ class ColumnParameter( RangeParameter ):
 
         if values is None and context:
             column_range = range( 1, ( context.columns or 0 ) + 1, 1 )
-            values = zip( column_range, column_range )
+            values = list(zip( column_range, column_range ))
         return RangeParameter.get_html( self, value, context=context, other_values=other_values, values=values, **kwd )
 
 
@@ -815,7 +816,7 @@ class JobExternalOutputMetadataWrapper( object ):
                 metadata_files.filename_override_metadata = abspath( tempfile.NamedTemporaryFile( dir=tmp_dir, prefix="metadata_override_%s_" % key ).name )
                 open( metadata_files.filename_override_metadata, 'wb+' )  # create the file on disk, so it cannot be reused by tempfile (unlikely, but possible)
                 override_metadata = []
-                for meta_key, spec_value in dataset.metadata.spec.iteritems():
+                for meta_key, spec_value in dataset.metadata.spec.items():
                     if isinstance( spec_value.param, FileParameter ) and dataset.metadata.get( meta_key, None ) is not None:
                         metadata_temp = MetadataTempFile()
                         shutil.copy( dataset.metadata.get( meta_key, None ).file_name, metadata_temp.file_name )
@@ -871,7 +872,7 @@ class JobExternalOutputMetadataWrapper( object ):
             sa_session.add( metadata_files )
             sa_session.flush()
 
-__all__ = [
+__all__ = (
     "Statement",
     "MetadataElement",
     "MetadataCollection",
@@ -889,4 +890,4 @@ __all__ = [
     "FileParameter",
     "MetadataTempFile",
     "JobExternalOutputMetadataWrapper",
-]
+)

lib/galaxy/model/orm/now.py

@@ -10,4 +10,4 @@ from datetime import datetime
 # Return the current time in UTC without any timezone information
 now = datetime.utcnow
 
-__all__ = [now]
+__all__ = ('now', )

lib/galaxy/tools/cwl/__init__.py

@@ -2,13 +2,12 @@ from .cwltool_deps import (
     needs_shell_quoting,
     shellescape,
 )
-from .parser import tool_proxy
-from .parser import workflow_proxy
+from .parser import tool_proxy, workflow_proxy
 from .representation import to_cwl_job, to_galaxy_parameters
 from .runtime_actions import handle_outputs
 
 
-__all__ = [
+__all__ = (
     'tool_proxy',
     'workflow_proxy',
     'handle_outputs',
@@ -16,4 +15,4 @@ __all__ = [
     'to_galaxy_parameters',
     'needs_shell_quoting',
     'shellescape',
-]
+)

lib/galaxy/tools/cwl/cwltool_deps.py

@@ -4,6 +4,7 @@ Use this as the import interface for cwltool and just call
 :func:`ensure_cwltool_available` before using any of the imported
 functionality at runtime.
 """
+import re
 
 try:
     import requests
@@ -40,8 +41,6 @@ except (ImportError, SyntaxError):
     # Drop SyntaxError once schema_salad supports Python 3
     schema_salad = None
 
-import re
-
 needs_shell_quoting = re.compile(r"""(^$|[\s|&;()<>\'"$@])""").search
 
 
@@ -65,7 +64,7 @@ def ensure_cwltool_available():
         raise ImportError(message)
 
 
-__all__ = [
+__all__ = (
     'main',
     'load_tool',
     'workflow',
@@ -74,4 +73,4 @@ __all__ = [
     'schema_salad',
     'shellescape',
     'needs_shell_quoting',
-]
+)

lib/galaxy/tools/cwl/parser.py

@@ -10,6 +10,8 @@ import logging
 import os
 from abc import ABCMeta, abstractmethod
 
+import six
+
 from galaxy.util import safe_makedirs
 from galaxy.util.bunch import Bunch
 from galaxy.util.odict import odict
@@ -119,8 +121,8 @@ def check_requirements(rec, tool=True):
             check_requirements(d, tool=tool)
 
 
+@six.add_metaclass(ABCMeta)
 class ToolProxy( object ):
-    __metaclass__ = ABCMeta
 
     def __init__(self, tool, tool_path):
         self._tool = tool
@@ -176,7 +178,7 @@ class CommandLineToolProxy(ToolProxy):
                 schema = self._tool.schemaDefs[schema_type]
 
             if schema["type"] == "record":
-                return map(_simple_field_to_input, schema["fields"])
+                return [_simple_field_to_input(_) for _ in schema["fields"]]
 
     def output_instances(self):
         outputs_schema = self._tool.outputs_record_schema
@@ -235,12 +237,12 @@ class JobProxy(object):
 
     def _ensure_cwl_job_initialized(self):
         if self._cwl_job is None:
-            self._cwl_job = self._tool_proxy._tool.job(
+            self._cwl_job = next(self._tool_proxy._tool.job(
                 self._input_dict,
                 self._output_callback,
                 basedir=self._job_directory,
                 use_container=False
-            ).next()
+            ))
             self._is_command_line_job = hasattr(self._cwl_job, "command_line")
 
     @property
@@ -536,7 +538,7 @@ class ConditionalInstance(object):
             when=odict(),
         )
         for value, block in self.whens:
-            as_dict["when"][value] = map(lambda i: i.to_dict(), block)
+            as_dict["when"][value] = [i.to_dict() for i in block]
 
         return as_dict
 
@@ -616,7 +618,7 @@ class OutputInstance(object):
         self.path = path
 
 
-__all__ = [
+__all__ = (
     'tool_proxy',
     'load_job_proxy',
-]
+)

lib/galaxy/tools/cwl/runtime_actions.py

@@ -20,7 +20,7 @@ def handle_outputs(job_directory=None):
     job_proxy = load_job_proxy(job_directory)
     tool_working_directory = os.path.join(job_directory, "working")
     outputs = job_proxy.collect_outputs(tool_working_directory)
-    for output_name, output in outputs.iteritems():
+    for output_name, output in outputs.items():
         target_path = job_proxy.output_path( output_name )
         if isinstance(output, dict) and "path" in output:
             output_path = output["path"]
@@ -48,6 +48,6 @@ def handle_outputs(job_directory=None):
             with open(target_path, "w") as f:
                 f.write(json.dumps(output))
 
-__all__ = [
+__all__ = (
     'handle_outputs',
-]
+)

lib/galaxy/tools/deps/commands.py

@@ -3,15 +3,12 @@ import os
 import subprocess
 import sys as _sys
 
+from six.moves import shlex_quote
+
 from galaxy.util import which
 
 STDOUT_INDICATOR = "-"
 
-try:
-    from shlex import quote as shell_quote
-except ImportError:
-    from pipes import quote as shell_quote
-
 
 def redirecting_io(sys=_sys):
     """Predicate to determine if we are redicting I/O in process."""
@@ -83,11 +80,11 @@ def argv_to_str(command_argv, quote=True):
 
     If None appears in the command list it is simply excluded.
 
-    Arguments are quoted with shlex.quote. That said, this method is not meant to be
+    Arguments are quoted with shlex_quote. That said, this method is not meant to be
     used in security critical paths of code and should not be used to sanitize
     code.
     """
-    map_func = shell_quote if quote else lambda x: x
+    map_func = shlex_quote if quote else lambda x: x
     return " ".join([map_func(c) for c in command_argv if c is not None])
 
 
@@ -144,7 +141,7 @@ class CommandLineException(Exception):
         return self.message
 
 
-__all__ = [
+__all__ = (
     'argv_to_str',
     'CommandLineException',
     'download_command',
@@ -153,6 +150,5 @@ __all__ = [
     'redirecting_io',
     'shell',
     'shell_process',
-    'shell_quote',
     'which',
-]
+)

lib/galaxy/tools/deps/conda_compat.py

@@ -6,8 +6,8 @@ PyPI. This module serves as a PyPI capable interface to these utilities.
 """
 import collections
 import os
-import yaml
 
+import yaml
 
 try:
     from conda_build.metadata import MetaData
@@ -15,7 +15,7 @@ except ImportError:
     MetaData = None
 
 try:
-    from anaconda_verify.recipe import render_jinja2, parse
+    from anaconda_verify.recipe import parse, render_jinja2
 except ImportError:
     render_jinja2 = None
     parse = None
@@ -106,7 +106,7 @@ class _MetaData(object):
 if MetaData is None:
     MetaData = _MetaData
 
-__all__ = [
+__all__ = (
     "MetaData",
     "raw_metadata",
-]
+)

lib/galaxy/tools/deps/conda_util.py

@@ -445,7 +445,7 @@ def filter_installed_targets(conda_targets, conda_context=None, verbose_install_
     installed = functools.partial(is_conda_target_installed,
                                   conda_context=conda_context,
                                   verbose_install_check=verbose_install_check)
-    return filter(installed, conda_targets)
+    return list(filter(installed, conda_targets))
 
 
 def build_isolated_environment(
@@ -510,7 +510,7 @@ def requirement_to_conda_targets(requirement, conda_context=None):
 def requirements_to_conda_targets(requirements, conda_context=None):
     r_to_ct = functools.partial(requirement_to_conda_targets,
                                 conda_context=conda_context)
-    conda_targets = map(r_to_ct, requirements)
+    conda_targets = (r_to_ct(_) for _ in requirements)
     return [c for c in conda_targets if c is not None]
 
 
@@ -520,10 +520,10 @@ def _ensure_conda_context(conda_context):
     return conda_context
 
 
-__all__ = [
+__all__ = (
     'CondaContext',
     'CondaTarget',
     'install_conda',
     'install_conda_target',
     'requirements_to_conda_targets',
-]
+)

lib/galaxy/tools/deps/container_resolvers/mulled.py

@@ -58,7 +58,7 @@ def list_cached_mulled_images(namespace=None):
 
         return image
 
-    return map(output_line_to_image, filter(name_filter, images_and_versions.splitlines()))
+    return [output_line_to_image(_) for _ in filter(name_filter, images_and_versions.splitlines())]
 
 
 def get_filter(namespace):
@@ -200,8 +200,8 @@ def mulled_targets(tool_info):
     return requirements_to_mulled_targets(tool_info.requirements)
 
 
-__all__ = [
+__all__ = (
     "CachedMulledContainerResolver",
     "MulledContainerResolver",
     "BuildMulledContainerResolver",
-]
+)

lib/galaxy/tools/deps/docker_util.py

@@ -3,7 +3,10 @@
 ...using common defaults and configuration mechanisms.
 """
 import os
-from .commands import argv_to_str, shell_quote
+
+from six.moves import shlex_quote
+
+from .commands import argv_to_str
 
 DEFAULT_DOCKER_COMMAND = "docker"
 DEFAULT_SUDO = True
@@ -33,7 +36,7 @@ class DockerVolume(object):
         if not volumes_as_str:
             return []
         volume_strs = [v.strip() for v in volumes_as_str.split(",")]
-        return map(DockerVolume.volume_from_str, volume_strs)
+        return [DockerVolume.volume_from_str(_) for _ in volume_strs]
 
     @staticmethod
     def volume_from_str(as_str):
@@ -152,19 +155,19 @@ def build_docker_run_command(
     if terminal:
         command_parts.append("-t")
     for env_directive in env_directives:
-        command_parts.extend(["-e", shell_quote(env_directive)])
+        command_parts.extend(["-e", shlex_quote(env_directive)])
     for volume in volumes:
-        command_parts.extend(["-v", shell_quote(str(volume))])
+        command_parts.extend(["-v", shlex_quote(str(volume))])
     if volumes_from:
-        command_parts.extend(["--volumes-from", shell_quote(str(volumes_from))])
+        command_parts.extend(["--volumes-from", shlex_quote(str(volumes_from))])
     if memory:
-        command_parts.extend(["-m", shell_quote(memory)])
+        command_parts.extend(["-m", shlex_quote(memory)])
     if name:
-        command_parts.extend(["--name", shell_quote(name)])
+        command_parts.extend(["--name", shlex_quote(name)])
     if working_directory:
-        command_parts.extend(["-w", shell_quote(working_directory)])
+        command_parts.extend(["-w", shlex_quote(working_directory)])
     if net:
-        command_parts.extend(["--net", shell_quote(net)])
+        command_parts.extend(["--net", shlex_quote(net)])
     if auto_rm:
         command_parts.append("--rm")
     if run_extra_arguments:
@@ -177,7 +180,7 @@ def build_docker_run_command(
     full_image = image
     if tag:
         full_image = "%s:%s" % (full_image, tag)
-    command_parts.append(shell_quote(full_image))
+    command_parts.append(shlex_quote(full_image))
     command_parts.append(container_command)
     return " ".join(command_parts)
 

lib/galaxy/tools/deps/mulled/_cli.py

@@ -14,6 +14,6 @@ def arg_parser(argv, globals):
     )
     return parser
 
-__all__ = [
-    "arg_parser"
-]
+__all__ = (
+    "arg_parser",
+)

lib/galaxy/tools/deps/mulled/mulled_build.py

@@ -14,17 +14,15 @@ import json
 import os
 import string
 import subprocess
-
 from sys import platform as _platform
 
-from galaxy.tools.deps import commands
-from galaxy.tools.deps import installable
-
 try:
     import yaml
 except ImportError:
     yaml = None
 
+from galaxy.tools.deps import commands, installable
+
 from ._cli import arg_parser
 from .util import build_target, conda_build_target_str, image_name
 from ..conda_compat import MetaData
@@ -236,7 +234,7 @@ def target_str_to_targets(targets_raw):
             target = build_target(target_str)
         return target
 
-    targets = map(parse_target, targets_raw.split(","))
+    targets = [parse_target(_) for _ in targets_raw.split(",")]
     return targets
 
 
@@ -281,7 +279,7 @@ def main(argv=None):
     mull_targets(targets, **args_to_mull_targets_kwds(args))
 
 
-__all__ = ["main"]
+__all__ = ("main", )
 
 
 if __name__ == '__main__':

lib/galaxy/tools/deps/mulled/mulled_build_channel.py

@@ -16,7 +16,7 @@ See recent changes that would be built with:
     mulled-build-channel list
 
 """
-
+from __future__ import print_function
 
 import os
 import time
@@ -75,7 +75,7 @@ def run_channel(args, build_last_n_versions=1):
 
 def get_pkg_names(args):
     """Print package names that would be affected."""
-    print('\n'.join([pkg_name for pkg_name, pkg_tests in get_affected_packages(args)]))
+    print('\n'.join(pkg_name for pkg_name, pkg_tests in get_affected_packages(args)))
 
 
 def add_channel_arguments(parser):
@@ -102,7 +102,7 @@ def main(argv=None):
         run_channel(args)
 
 
-__all__ = ["main"]
+__all__ = ("main", )
 
 
 if __name__ == '__main__':

lib/galaxy/tools/deps/mulled/mulled_build_files.py

@@ -17,7 +17,6 @@ import glob
 import os
 
 from ._cli import arg_parser
-
 from .mulled_build import (
     add_build_arguments,
     args_to_mull_targets_kwds,
@@ -73,7 +72,7 @@ def _parse_line(line_str):
     return _Line(*line_parts)
 
 
-__all__ = ["main"]
+__all__ = ("main", )
 
 
 if __name__ == '__main__':

lib/galaxy/tools/deps/mulled/mulled_build_tool.py

@@ -40,12 +40,12 @@ def requirements_to_mulled_targets(requirements):
 
     Only package requirements are retained.
     """
-    package_requirements = filter(lambda r: r.type == "package", requirements)
-    targets = map(lambda r: build_target(r.name, r.version), package_requirements)
+    package_requirements = [r for r in requirements if r.type == "package"]
+    targets = [build_target(r.name, r.version) for r in package_requirements]
     return targets
 
 
-__all__ = ["main", "requirements_to_mulled_targets"]
+__all__ = ("main", "requirements_to_mulled_targets")
 
 
 if __name__ == '__main__':

lib/galaxy/tools/deps/mulled/util.py

@@ -105,7 +105,7 @@ def image_name(targets, image_build=None, name_override=None):
         return "mulled-v1-%s%s" % (m.hexdigest(), suffix)
 
 
-__all__ = [
+__all__ = (
     "build_target",
     "conda_build_target_str",
     "image_name",
@@ -114,4 +114,4 @@ __all__ = [
     "split_tag",
     "Target",
     "version_sorted",
-]
+)

lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py

@@ -5,7 +5,6 @@ via shed2tap (e.g. https://github.com/jmchilton/homebrew-toolshed).
 """
 import logging
 import os
-
 from xml.etree import ElementTree as ET
 
 from .resolver_mixins import (
@@ -147,4 +146,4 @@ def build_recipe_name(package_name, package_version, repository_owner, repositor
     return base
 
 
-__all__ = ['HomebrewToolShedDependencyResolver']
+__all__ = ('HomebrewToolShedDependencyResolver', )

lib/galaxy/tools/deps/resolvers/conda.py

@@ -244,4 +244,4 @@ def _string_as_bool( value ):
     return str( value ).lower() == "true"
 
 
-__all__ = ['CondaDependencyResolver']
+__all__ = ('CondaDependencyResolver', )

lib/galaxy/tools/deps/resolvers/galaxy_packages.py

@@ -122,4 +122,8 @@ def _is_dependency_directory(directory):
     return exists(join(directory, 'env.sh')) or exists(join(directory, 'bin'))
 
 
-__all__ = ['GalaxyPackageDependencyResolver', 'GalaxyPackageDependency', 'ToolShedDependency']
+__all__ = (
+    'GalaxyPackageDependency',
+    'GalaxyPackageDependencyResolver',
+    'ToolShedDependency'
+)

lib/galaxy/tools/deps/resolvers/homebrew.py

@@ -53,4 +53,4 @@ def _string_as_bool( value ):
     return str( value ).lower() == "true"
 
 
-__all__ = ['HomebrewDependencyResolver']
+__all__ = ('HomebrewDependencyResolver', )

lib/galaxy/tools/deps/resolvers/modules.py

@@ -7,7 +7,6 @@ it, hence support for it will be minimal. The Galaxy team eagerly welcomes
 community contribution and maintenance however.
 """
 import logging
-
 from os import environ, pathsep
 from os.path import exists, isdir, join
 from subprocess import PIPE, Popen
@@ -180,4 +179,4 @@ class ModuleDependency(Dependency):
 def _string_as_bool( value ):
     return str( value ).lower() == "true"
 
-__all__ = ['ModuleDependencyResolver']
+__all__ = ('ModuleDependencyResolver', )

lib/galaxy/tools/deps/resolvers/tool_shed_packages.py

@@ -2,7 +2,6 @@ from os.path import abspath, exists, join
 
 from .galaxy_packages import BaseGalaxyPackageDependencyResolver, ToolShedDependency
 from .resolver_mixins import UsesInstalledRepositoriesMixin
-
 from ..resolvers import NullDependency
 
 
@@ -64,4 +63,4 @@ class ToolShedPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, Use
         return NullDependency(version=None, name=name)
 
 
-__all__ = ['ToolShedPackageDependencyResolver']
+__all__ = ('ToolShedPackageDependencyResolver', )

lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py

@@ -19,7 +19,6 @@ See bottom for instructions on how to add this resolver.
 
 """
 import logging
-
 from os import listdir
 from os.path import exists, getmtime, join
 
@@ -153,7 +152,7 @@ class CandidateDependency(Dependency):
         return self.dependency.shell_commands( requirement )
 
 
-__all__ = ['UnlinkedToolShedPackageDependencyResolver']
+__all__ = ('UnlinkedToolShedPackageDependencyResolver', )
 
 """
 At the time of writing July 3 2015 this resolver has to be plugged in.

lib/galaxy/tools/exception_handling.py

@@ -1,6 +1,6 @@
 # We put a tool that references this package into the tool shed
 # so we have to provide this legacy location for import indefinitely
 # it seems.
-from galaxy.util.ucsc import UCSCOutWrapper, UCSCLimitException
+from galaxy.util.ucsc import UCSCLimitException, UCSCOutWrapper
 
-__all__ = ['UCSCOutWrapper', 'UCSCLimitException']
+__all__ = ('UCSCOutWrapper', 'UCSCLimitException')

lib/galaxy/tools/execute.py

@@ -4,13 +4,15 @@ from various states, tracking results, and building implicit dataset
 collections from matched collections.
 """
 import collections
+import logging
+from threading import Thread
+
+from six.moves.queue import Queue
+
+from galaxy.tools.actions import on_text_for_names, ToolExecutionCache
 from galaxy.tools.parser import ToolOutputCollectionPart
 from galaxy.util import ExecutionTimer
-from galaxy.tools.actions import on_text_for_names, ToolExecutionCache
-from threading import Thread
-from Queue import Queue
 
-import logging
 log = logging.getLogger( __name__ )
 
 EXECUTION_SUCCESS_MESSAGE = "Tool [%s] created job [%s] %s"
@@ -135,7 +137,6 @@ class ToolExecutionTracker( object ):
             return []
 
         structure = self.collection_info.structure
-        collections = self.collection_info.collections.values()
 
         # params is just one sample tool param execution with parallelized
         # collection replaced with a specific dataset. Need to replace this
@@ -143,13 +144,13 @@ class ToolExecutionTracker( object ):
         # label.
         params.update( self.collection_info.collections )  # Replace datasets with source collections for labelling outputs.
 
-        collection_names = map( lambda c: "collection %d" % c.hid, collections )
+        collection_names = ["collection %d" % c.hid for c in self.collection_info.collections.values()]
         on_text = on_text_for_names( collection_names )
 
         collections = {}
 
-        implicit_inputs = list(self.collection_info.collections.iteritems())
-        for output_name, outputs in self.outputs_by_output_name.iteritems():
+        implicit_inputs = list(self.collection_info.collections.items())
+        for output_name, outputs in self.outputs_by_output_name.items():
             if not len( structure ) == len( outputs ):
                 # Output does not have the same structure, if all jobs were
                 # successfully submitted this shouldn't have happened.
@@ -202,4 +203,4 @@ class ToolExecutionTracker( object ):
         trans.sa_session.flush()
         self.implicit_collections = collections
 
-__all__ = [ execute ]
+__all__ = ( 'execute', )

lib/galaxy/tools/loader.py

@@ -7,4 +7,4 @@ from galaxy.util.xml_macros import (
 
 load_tool = load
 
-__all__ = ["load_tool", "raw_tool_xml_tree", "imported_macro_paths", "template_macro_params"]
+__all__ = ("load_tool", "raw_tool_xml_tree", "imported_macro_paths", "template_macro_params")

lib/galaxy/tools/loader_directory.py

@@ -241,11 +241,11 @@ def _find_tool_files(path, recursive, enable_beta_formats):
                 files = glob.glob(path + "/*.xml")
             else:
                 files = _find_files(path, "*.xml")
-        return map(os.path.abspath, files)
+        return [os.path.abspath(_) for _ in files]
 
 
 def _has_extension(path, extensions):
-    return any(map(lambda e: path.endswith(e), extensions))
+    return any(path.endswith(e) for e in extensions)
 
 
 def _find_files(directory, pattern='*'):
@@ -266,7 +266,7 @@ BETA_TOOL_CHECKERS = {
     'cwl': looks_like_a_tool_cwl,
 }
 
-__all__ = [
+__all__ = (
     "find_possible_tools_from_path",
     "is_a_yaml_with_class",
     "is_tool_load_error",
@@ -276,4 +276,4 @@ __all__ = [
     "looks_like_a_tool_cwl",
     "looks_like_a_tool_xml",
     "looks_like_a_tool_yaml",
-]
+)

lib/galaxy/tools/parameters/__init__.py

@@ -4,15 +4,16 @@ Classes encapsulating Galaxy tool parameters.
 import re
 from json import dumps, loads
 
-from basic import RuntimeValue, DataCollectionToolParameter, DataToolParameter, SelectToolParameter
-from grouping import Conditional, Repeat, Section, UploadDataset
 from galaxy.util.expressions import ExpressionContext
 from galaxy.util.json import json_fix
 
+from .basic import DataCollectionToolParameter, DataToolParameter, RuntimeValue, SelectToolParameter
+from .grouping import Conditional, Repeat, Section, UploadDataset
+
 REPLACE_ON_TRUTHY = object()
 
 # Some tools use the code tag and access the code base, expecting certain tool parameters to be available here.
-__all__ = [ DataCollectionToolParameter, DataToolParameter, SelectToolParameter ]
+__all__ = ( 'DataCollectionToolParameter', 'DataToolParameter', 'SelectToolParameter' )
 
 
 def visit_input_values( inputs, input_values, callback, name_prefix='', label_prefix='', parent_prefix='', context=None, no_replacement_value=REPLACE_ON_TRUTHY ):
@@ -82,7 +83,7 @@ def visit_input_values( inputs, input_values, callback, name_prefix='', label_pr
 
     context = ExpressionContext( input_values, context )
     payload = { 'context': context, 'no_replacement_value': no_replacement_value }
-    for input in inputs.itervalues():
+    for input in inputs.values():
         if isinstance( input, Repeat ) or isinstance( input, UploadDataset ):
             values = input_values[ input.name ] = input_values.get( input.name, [] )
             for i, d in enumerate( values ):
@@ -143,7 +144,7 @@ def params_to_strings( params, param_values, app ):
     such).
     """
     rval = dict()
-    for key, value in param_values.iteritems():
+    for key, value in param_values.items():
         if key in params:
             value = params[ key ].value_to_basic( value, app )
         rval[ key ] = str( dumps( value ) )
@@ -158,7 +159,7 @@ def params_from_strings( params, param_values, app, ignore_errors=False ):
     preferred form).
     """
     rval = dict()
-    for key, value in param_values.iteritems():
+    for key, value in param_values.items():
         value = json_fix( loads( value ) )
         if key in params:
             value = params[ key ].value_from_basic( value, app, ignore_errors )
@@ -173,7 +174,7 @@ def params_to_incoming( incoming, inputs, input_values, app, name_prefix="" ):
 
     Useful for e.g. the rerun function.
     """
-    for input in inputs.itervalues():
+    for input in inputs.values():
         if isinstance( input, Repeat ) or isinstance( input, UploadDataset ):
             for d in input_values[ input.name ]:
                 index = d[ '__index__' ]

lib/galaxy/tools/parameters/dataset_matcher.py

@@ -1,6 +1,7 @@
+from logging import getLogger
+
 import galaxy.model
 
-from logging import getLogger
 log = getLogger( __name__ )
 
 ROLES_UNSET = object()
@@ -179,4 +180,4 @@ class DatasetCollectionMatcher( object ):
                 break
         return valid
 
-__all__ = [ DatasetMatcher, DatasetCollectionMatcher ]
+__all__ = ( 'DatasetMatcher', 'DatasetCollectionMatcher' )

lib/galaxy/tools/parameters/wrapped.py

@@ -1,20 +1,21 @@
 from galaxy.tools.parameters.basic import (
-    DataToolParameter,
     DataCollectionToolParameter,
+    DataToolParameter,
     SelectToolParameter
 )
-from galaxy.tools.wrappers import (
-    InputValueWrapper,
-    SelectToolParameterWrapper,
-    DatasetFilenameWrapper,
-    DatasetListWrapper,
-    DatasetCollectionWrapper
-)
 from galaxy.tools.parameters.grouping import (
-    Repeat,
     Conditional,
+    Repeat,
     Section
 )
+from galaxy.tools.wrappers import (
+    DatasetCollectionWrapper,
+    DatasetFilenameWrapper,
+    DatasetListWrapper,
+    InputValueWrapper,
+    SelectToolParameterWrapper
+)
+
 PARAMS_UNWRAPPED = object()
 
 
@@ -40,7 +41,7 @@ class WrappedParameters( object ):
         incoming = self.incoming
 
         # Wrap tool inputs as necessary
-        for input in inputs.itervalues():
+        for input in inputs.values():
             if input.name not in input_values and skip_missing_values:
                 continue
             value = input_values[ input.name ]
@@ -112,4 +113,4 @@ def make_list_copy( from_list ):
     return new_list
 
 
-__all__ = [ 'WrappedParameters', 'make_dict_copy' ]
+__all__ = ( 'WrappedParameters', 'make_dict_copy' )

lib/galaxy/tools/parameters/wrapped_json.py

@@ -1,6 +1,6 @@
 import logging
-log = logging.getLogger(__name__)
 
+log = logging.getLogger(__name__)
 
 SKIP_INPUT = object()
 
@@ -9,7 +9,7 @@ def json_wrap(inputs, input_values, as_dict=None, handle_files="SKIP"):
     if as_dict is None:
         as_dict = {}
 
-    for input in inputs.itervalues():
+    for input in inputs.values():
         input_name = input.name
         value = input_values[input_name]
         json_value = _json_wrap_input(input, value, handle_files=handle_files)
@@ -65,7 +65,7 @@ def _json_wrap_input(input, value, handle_files="SKIP"):
         json_value = _cast_if_not_none(value, bool)
     elif input_type == "data_column":
         # value is a SelectToolParameterWrapper()
-        json_value = map(int, _cast_if_not_none(value.value, list))
+        json_value = [int(_) for _ in _cast_if_not_none(value.value, list)]
     else:
         raise NotImplementedError("input_type [%s] not implemented" % input_type)
 
@@ -80,4 +80,4 @@ def _cast_if_not_none(value, cast_to, empty_to_none=False):
         return cast_to(value)
 
 
-__all__ = ['json_wrap']
+__all__ = ('json_wrap', )

lib/galaxy/tools/parser/__init__.py

@@ -1,15 +1,14 @@
 """ Package responsible for parsing tools from files/abstract tool sources.
 """
-from .factory import get_input_source
-from .factory import get_tool_source
+from .factory import get_input_source, get_tool_source
 from .interface import ToolSource
 from .output_objects import (
     ToolOutputCollectionPart,
 )
 
-__all__ = [
+__all__ = (
     "get_input_source",
     "get_tool_source",
     "ToolOutputCollectionPart",
     "ToolSource",
-]
+)

lib/galaxy/tools/parser/factory.py

@@ -10,8 +10,7 @@ from galaxy.util.odict import odict
 
 from .cwl import CwlToolSource
 from .interface import InputSource
-from .xml import XmlInputSource
-from .xml import XmlToolSource
+from .xml import XmlInputSource, XmlToolSource
 from .yaml import YamlToolSource
 
 log = logging.getLogger(__name__)
@@ -72,4 +71,4 @@ def get_input_source(content):
     return content
 
 
-__all__ = ["get_tool_source", "get_input_source"]
+__all__ = ("get_tool_source", "get_input_source")

lib/galaxy/tools/toolbox/__init__.py

@@ -1,16 +1,13 @@
 """API for this module containing functionality related to the toolbox."""
 
-from .base import AbstractToolBox
-from .base import BaseGalaxyToolBox
+from .base import AbstractToolBox, BaseGalaxyToolBox
 
-from .panel import panel_item_types
-from .panel import ToolSection
-from .panel import ToolSectionLabel
+from .panel import panel_item_types, ToolSection, ToolSectionLabel
 
-__all__ = [
+__all__ = (
     "AbstractToolBox",
     "BaseGalaxyToolBox",
     "panel_item_types",
     "ToolSection",
     "ToolSectionLabel",
-]
+)

lib/galaxy/tools/toolbox/lineages/__init__.py

@@ -3,4 +3,4 @@ from .interface import ToolLineage
 from .tool_shed import ToolVersionCache
 
 
-__all__ = ["LineageMap", "ToolLineage", "ToolVersionCache"]
+__all__ = ("LineageMap", "ToolLineage", "ToolVersionCache")

lib/galaxy/tools/toolbox/lineages/factory.py

@@ -46,4 +46,4 @@ class LineageMap(object):
         return self.lineage_map.get(versionless_tool_id, None)
 
 
-__all__ = ["LineageMap"]
+__all__ = ("LineageMap", )

lib/galaxy/tools/toolbox/lineages/tool_shed.py

@@ -1,5 +1,4 @@
-from .interface import ToolLineage
-from .interface import ToolLineageVersion
+from .interface import ToolLineage, ToolLineageVersion
 
 try:
     from galaxy.model.tool_shed_install import ToolVersion
@@ -78,13 +77,13 @@ class ToolShedLineage(ToolLineage):
         return result
 
     def get_versions( self, reverse=False ):
-        return map( ToolLineageVersion.from_guid, self.get_version_ids( reverse=reverse ) )
+        return [ ToolLineageVersion.from_guid(_) for _ in self.get_version_ids( reverse=reverse ) ]
 
     def to_dict(self):
         tool_shed_repository = self._tool_shed_repository
         rval = dict(
             tool_version_id=self.tool_version_id,
-            tool_versions=map(lambda v: v.to_dict(), self.get_versions()),
+            tool_versions=[v.to_dict() for v in self.get_versions()],
             tool_shed_repository=tool_shed_repository if tool_shed_repository is not None else None,
             lineage_type='tool_shed',
         )
@@ -94,4 +93,4 @@ class ToolShedLineage(ToolLineage):
 def get_installed_tool_version( app, tool_id ):
     return app.tool_version_cache.tool_version_by_tool_id.get(tool_id, None)
 
-__all__ = [ "ToolShedLineage" ]
+__all__ = ( "ToolShedLineage", )

lib/galaxy/tools/toolbox/parser.py

@@ -3,9 +3,9 @@
 These files define tool lists, sections, labels, etc... the elements of the
 Galaxy tool panel.
 """
-from abc import ABCMeta
-from abc import abstractmethod
+from abc import ABCMeta, abstractmethod
 
+import six
 import yaml
 
 from galaxy.util import parse_xml, string_as_bool
@@ -13,11 +13,10 @@ from galaxy.util import parse_xml, string_as_bool
 DEFAULT_MONITOR = False
 
 
+@six.add_metaclass(ABCMeta)
 class ToolConfSource(object):
     """Interface represents a container of tool references."""
 
-    __metaclass__ = ABCMeta
-
     @abstractmethod
     def parse_items(self):
         """Return a list of ToolConfItem describing source."""
@@ -45,7 +44,7 @@ class XmlToolConfSource(ToolConfSource):
         return self.root.get('tool_path')
 
     def parse_items(self):
-        return map(ensure_tool_conf_item, self.root.getchildren())
+        return [ensure_tool_conf_item(_) for _ in self.root.getchildren()]
 
     def is_shed_tool_conf(self):
         has_tool_path = self.parse_tool_path() is not None
@@ -67,7 +66,7 @@ class YamlToolConfSource(ToolConfSource):
         return self.as_dict.get('tool_path')
 
     def parse_items(self):
-        return map(ToolConfItem.from_dict, self.as_dict.get('items'))
+        return [ToolConfItem.from_dict(_) for _ in self.as_dict.get('items')]
 
     def parse_monitor(self):
         return self.as_dict.get('monitor', DEFAULT_MONITOR)
@@ -94,7 +93,7 @@ class ToolConfItem(object):
         del as_dict['type']
         attributes = as_dict
         if type == 'section':
-            items = map(cls.from_dict, as_dict['items'])
+            items = [cls.from_dict(_) for _ in as_dict['items']]
             del as_dict['items']
             item = ToolConfSection(attributes, items)
         else:
@@ -141,18 +140,18 @@ def ensure_tool_conf_item(xml_or_item):
         if type != "section":
             return ToolConfItem(type, attributes, elem)
         else:
-            items = map(ensure_tool_conf_item, elem.getchildren())
+            items = [ensure_tool_conf_item(_) for _ in elem.getchildren()]
             return ToolConfSection(attributes, items, elem=elem)
 
 
 def get_toolbox_parser(config_filename):
-    is_yaml = any(map(lambda e: config_filename.endswith(e), [".yml", ".yaml", ".json"]))
+    is_yaml = any(config_filename.endswith(e) for e in [".yml", ".yaml", ".json"])
     if is_yaml:
         return YamlToolConfSource(config_filename)
     else:
         return XmlToolConfSource(config_filename)
 
-__all__ = [
+__all__ = (
     "get_toolbox_parser",
     "ensure_tool_conf_item",
-]
+)

lib/galaxy/tools/xsd/galaxy.xsd

@@ -1990,6 +1990,33 @@ multiple formats.
        help="The set of aligned reads." />
 ```
 
+Perhaps counter-intuitively, a ``multiple="true"`` data parameter requires at least one
+data input. If ``optional="true"`` is specified, this condition is relaxed and the user
+is allowed to select 0 datasets. Unfortunately, if 0 datasets are selected the resulting
+value for the parameter during Cheetah templating (such as in a ``command`` block) will
+effectively be a list with one ``None``-like entity in it.
+
+The following idiom can be used to iterate over such a list and build a hypothetical ``-B``
+parameter for each file - the ``if`` block is used to handle the case where a ``None``-like
+entity appears in the list because no files were selected: 
+
+```
+#for $input in $input1
+    #if $input
+        -B "$input"
+    #end if
+#end for
+```
+
+Some example tools using ``multiple="true"`` data parameters include:
+
+- [multi_data_param.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/multi_data_param.xml)
+- [multi_data_optional.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/multi_data_optional.xml)
+
+Additionally, a detailed discussion of handling multiple homogenous files can be found in the
+the [Planemo Documentation](http://planemo.readthedocs.io/en/latest/writing_advanced.html#consuming-collections)
+on this topic.
+
 $attribute_list:format,multiple
 
 #### ``select``

lib/galaxy/workflow/extract.py

@@ -1,25 +1,26 @@
 """ This module contains functionality to aid in extracting workflows from
 histories.
 """
-from galaxy.util.odict import odict
-from galaxy import exceptions
-from galaxy import model
+import logging
+
+from galaxy import exceptions, model
 from galaxy.tools.parameters.basic import (
-    DataToolParameter,
-    DataCollectionToolParameter
+    DataCollectionToolParameter,
+    DataToolParameter
 )
-from galaxy.tools.parser import ToolOutputCollectionPart
 from galaxy.tools.parameters.grouping import (
     Conditional,
     Repeat,
     Section
 )
+from galaxy.tools.parser import ToolOutputCollectionPart
+from galaxy.util.odict import odict
+
 from .steps import (
     attach_ordered_steps,
     order_workflow_steps_with_levels
 )
 
-import logging
 log = logging.getLogger( __name__ )
 
 WARNING_SOME_DATASETS_NOT_READY = "Some datasets still queued or running were ignored"
@@ -67,16 +68,14 @@ def extract_steps( trans, history=None, job_ids=None, dataset_ids=None, dataset_
     elif type( dataset_collection_ids) is not list:
         dataset_collection_ids = [  dataset_collection_ids ]
     # Convert both sets of ids to integers
-    job_ids = [ int( id ) for id in job_ids ]
-    dataset_ids = [ int( id ) for id in dataset_ids ]
-    dataset_collection_ids = [ int( id ) for id in dataset_collection_ids ]
-    # Find each job, for security we (implicately) check that they are
-    # associated witha job in the current history.
+    job_ids = [ int( _ ) for _ in job_ids ]
+    dataset_ids = [ int( _ ) for _ in dataset_ids ]
+    dataset_collection_ids = [ int( _ ) for _ in dataset_collection_ids ]
+    # Find each job, for security we (implicitly) check that they are
+    # associated with a job in the current history.
     summary = WorkflowSummary( trans, history )
     jobs = summary.jobs
-    jobs_by_id = dict( ( job.id, job ) for job in jobs.keys() )
     steps = []
-    steps_by_job_id = {}
     hid_to_output_pair = {}
     # Input dataset steps
     for i, hid in enumerate( dataset_ids ):
@@ -104,10 +103,10 @@ def extract_steps( trans, history=None, job_ids=None, dataset_ids=None, dataset_
         steps.append( step )
     # Tool steps
     for job_id in job_ids:
-        if job_id not in jobs_by_id:
-            log.warning( "job_id %s not found in jobs_by_id %s" % ( job_id, jobs_by_id ) )
+        if job_id not in summary.job_id2representative_job:
+            log.warning( "job_id %s not found in job_id2representative_job %s" % ( job_id, summary.job_id2representative_job ) )
             raise AssertionError( "Attempt to create workflow with job not connected to current history" )
-        job = jobs_by_id[ job_id ]
+        job = summary.job_id2representative_job[job_id]
         tool_inputs, associations = step_inputs( trans, job )
         step = model.WorkflowStep()
         step.type = 'tool'
@@ -134,7 +133,6 @@ def extract_steps( trans, history=None, job_ids=None, dataset_ids=None, dataset_
                 conn.output_step = other_step
                 conn.output_name = other_name
         steps.append( step )
-        steps_by_job_id[ job_id ] = step
         # Store created dataset hids
         for assoc in (job.output_datasets + job.output_dataset_collection_instances):
             assoc_name = assoc.name
@@ -147,8 +145,8 @@ def extract_steps( trans, history=None, job_ids=None, dataset_ids=None, dataset_
                     if query_assoc_name == assoc_name:
                         hid = dataset_collection.hid
                 if hid is None:
-                    template = "Failed to find matching implicit job - job is %s, jobs are %s, assoc_name is %s."
-                    message = template % ( job.id, jobs, assoc.name )
+                    template = "Failed to find matching implicit job - job id is %s, implicit pairs are %s, assoc_name is %s."
+                    message = template % ( job.id, jobs[job], assoc_name )
                     log.warning( message )
                     raise Exception( "Failed to extract job." )
             else:
@@ -202,6 +200,7 @@ class WorkflowSummary( object ):
         self.history = history
         self.warnings = set()
         self.jobs = odict()
+        self.job_id2representative_job = {}  # map a non-fake job id to its representative job
         self.implicit_map_jobs = []
         self.collection_types = {}
 
@@ -223,25 +222,27 @@ class WorkflowSummary( object ):
         else:
             self.__summarize_dataset( content )
 
-    def __summarize_dataset_collection( self, content ):
-        content = self.__original_hdca( content )
-        dataset_collection = content
-        hid = content.hid
-        self.collection_types[ hid ] = content.collection.collection_type
-        cja = content.creating_job_associations
+    def __summarize_dataset_collection( self, dataset_collection ):
+        dataset_collection = self.__original_hdca( dataset_collection )
+        hid = dataset_collection.hid
+        self.collection_types[ hid ] = dataset_collection.collection.collection_type
+        cja = dataset_collection.creating_job_associations
         if cja:
-            # Use the first job to represent all mapped jobs.
-            representive_job_assoc = content.creating_job_associations[0]
-            job = representive_job_assoc.job
-            if job not in self.jobs or self.jobs[ job ][ 0 ][ 1 ].history_content_type == "dataset":
-                self.jobs[ job ] = [ ( representive_job_assoc.name, dataset_collection ) ]
-                if content.implicit_output_name:
-                    self.implicit_map_jobs.append( job )
+            # Use the "first" job to represent all mapped jobs.
+            representative_assoc = cja[0]
+            representative_job = representative_assoc.job
+            if representative_job not in self.jobs or self.jobs[ representative_job ][ 0 ][ 1 ].history_content_type == "dataset":
+                self.jobs[ representative_job ] = [ ( representative_assoc.name, dataset_collection ) ]
+                if dataset_collection.implicit_output_name:
+                    self.implicit_map_jobs.append( representative_job )
             else:
-                self.jobs[ job ].append( ( representive_job_assoc.name, dataset_collection ) )
+                self.jobs[ representative_job ].append( ( representative_assoc.name, dataset_collection ) )
+            for assoc in cja:
+                job = assoc.job
+                self.job_id2representative_job[job.id] = representative_job
         # This whole elif condition may no longer be needed do to additional
         # tracking with creating_job_associations. Will delete at some point.
-        elif content.implicit_output_name:
+        elif dataset_collection.implicit_output_name:
             # TODO: Optimize db call
             dataset_instance = dataset_collection.collection.dataset_instances[ 0 ]
             if not self.__check_state( dataset_instance ):
@@ -249,38 +250,40 @@ class WorkflowSummary( object ):
                 # makes me wonder if even need this check at all?
                 return
 
-            job_hda = self.__original_hda( dataset_instance )
-            if not job_hda.creating_job_associations:
+            original_hda = self.__original_hda( dataset_instance )
+            if not original_hda.creating_job_associations:
                 log.warning( "An implicitly create output dataset collection doesn't have a creating_job_association, should not happen!" )
                 job = DatasetCollectionCreationJob( dataset_collection )
                 self.jobs[ job ] = [ ( None, dataset_collection ) ]
 
-            for assoc in job_hda.creating_job_associations:
+            for assoc in original_hda.creating_job_associations:
                 job = assoc.job
                 if job not in self.jobs or self.jobs[ job ][ 0 ][ 1 ].history_content_type == "dataset":
                     self.jobs[ job ] = [ ( assoc.name, dataset_collection ) ]
+                    self.job_id2representative_job[job.id] = job
                     self.implicit_map_jobs.append( job )
                 else:
                     self.jobs[ job ].append( ( assoc.name, dataset_collection ) )
         else:
-            job = DatasetCollectionCreationJob( content )
-            self.jobs[ job ] = [ ( None, content ) ]
+            job = DatasetCollectionCreationJob( dataset_collection )
+            self.jobs[ job ] = [ ( None, dataset_collection ) ]
 
     def __summarize_dataset( self, dataset ):
         if not self.__check_state( dataset ):
             return
 
-        job_hda = self.__original_hda( dataset )
+        original_hda = self.__original_hda( dataset )
 
-        if not job_hda.creating_job_associations:
+        if not original_hda.creating_job_associations:
             self.jobs[ FakeJob( dataset ) ] = [ ( None, dataset ) ]
 
-        for assoc in job_hda.creating_job_associations:
+        for assoc in original_hda.creating_job_associations:
             job = assoc.job
             if job in self.jobs:
                 self.jobs[ job ].append( ( assoc.name, dataset ) )
             else:
                 self.jobs[ job ] = [ ( assoc.name, dataset ) ]
+                self.job_id2representative_job[job.id] = job
 
     def __original_hdca( self, hdca ):
         while hdca.copied_from_history_dataset_collection_association:
@@ -288,11 +291,10 @@ class WorkflowSummary( object ):
         return hdca
 
     def __original_hda( self, hda ):
-        # if this hda was copied from another, we need to find the job that created the origial hda
-        job_hda = hda
-        while job_hda.copied_from_history_dataset_association:
-            job_hda = job_hda.copied_from_history_dataset_association
-        return job_hda
+        # if this hda was copied from another, we need to find the job that created the original hda
+        while hda.copied_from_history_dataset_association:
+            hda = hda.copied_from_history_dataset_association
+        return hda
 
     def __check_state( self, hda ):
         # FIXME: Create "Dataset.is_finished"
@@ -369,4 +371,4 @@ def __cleanup_param_values( inputs, values ):
     cleanup( "", inputs, values )
     return associations
 
-__all__ = [ summarize, extract_workflow ]
+__all__ = ( 'summarize', 'extract_workflow' )

lib/galaxy/workflow/run.py

@@ -1,15 +1,13 @@
+import logging
 import uuid
 
-from galaxy import model
-from galaxy import util
-
-from galaxy.util.odict import odict
+from galaxy import model, util
 from galaxy.util import ExecutionTimer
+from galaxy.util.odict import odict
 from galaxy.workflow import modules
-from galaxy.workflow.run_request import WorkflowRunConfig
-from galaxy.workflow.run_request import workflow_run_config_to_request
+from galaxy.workflow.run_request import (workflow_run_config_to_request,
+    WorkflowRunConfig)
 
-import logging
 log = logging.getLogger( __name__ )
 
 
@@ -253,7 +251,7 @@ class WorkflowProgress( object ):
                 self.module_injector.inject( step )
                 if step_id not in step_states:
                     template = "Workflow invocation [%s] has no step state for step id [%s]. States ids are %s."
-                    message = template % (self.workflow_invocation.id, step_id, step_states.keys())
+                    message = template % (self.workflow_invocation.id, step_id, list(step_states.keys()))
                     raise Exception(message)
                 runtime_state = step_states[ step_id ].value
                 step.state = step.module.recover_runtime_state( runtime_state )
@@ -399,4 +397,4 @@ class WorkflowProgress( object ):
         except modules.DelayedWorkflowEvaluation:
             self.mark_step_outputs_delayed( step )
 
-__all__ = [ invoke, WorkflowRunConfig ]
+__all__ = ( 'invoke', 'WorkflowRunConfig' )

lib/galaxy/workflow/schedulers/core.py

@@ -1,14 +1,13 @@
 """ The class defines the stock Galaxy workflow scheduling plugin - currently
 it simply schedules the whole workflow up front when offered.
 """
-from ..schedulers import ActiveWorkflowSchedulingPlugin
+import logging
 
 from galaxy.work import context
+from galaxy.workflow import run, run_request
 
-from galaxy.workflow import run
-from galaxy.workflow import run_request
+from ..schedulers import ActiveWorkflowSchedulingPlugin
 
-import logging
 log = logging.getLogger( __name__ )
 
 
@@ -43,4 +42,4 @@ class CoreWorkflowSchedulingPlugin( ActiveWorkflowSchedulingPlugin ):
             workflow_invocation=workflow_invocation,
         )
 
-__all__ = [ 'CoreWorkflowSchedulingPlugin' ]
+__all__ = ( 'CoreWorkflowSchedulingPlugin', )

test/api/test_workflow_extraction.py

@@ -216,7 +216,7 @@ test_data:
   text_input2: "samp1\t30.0\nsamp2\t40.0\n"
 """)
         tool_ids = [ "cat1", "collection_split_on_column", "cat_list" ]
-        job_ids = list(map( functools.partial(self._job_id_for_tool, jobs_summary.jobs ), tool_ids ))
+        job_ids = [ functools.partial(self._job_id_for_tool, jobs_summary.jobs )(_) for _ in tool_ids ]
         downloaded_workflow = self._extract_and_download_workflow(
             dataset_ids=[ "1", "2" ],
             job_ids=job_ids,
@@ -266,7 +266,7 @@ test_data:
         content: "samp1\t30.0\nsamp2\t40.0\n"
 """)
         tool_ids = [ "cat1", "collection_creates_pair", "cat_collection", "cat_list" ]
-        job_ids = list(map( functools.partial(self._job_id_for_tool, jobs_summary.jobs ), tool_ids ))
+        job_ids = [ functools.partial(self._job_id_for_tool, jobs_summary.jobs )(_) for _ in tool_ids ]
         downloaded_workflow = self._extract_and_download_workflow(
             dataset_collection_ids=[ "3" ],
             job_ids=job_ids,

test/api/workflows_format_2/__init__.py

@@ -5,7 +5,7 @@ from .interface import ImporterGalaxyInterface
 from .main import convert_and_import_workflow
 
 
-__all__ = [
+__all__ = (
     'convert_and_import_workflow',
     'ImporterGalaxyInterface',
-]
+)

test/api/workflows_format_2/converter.py

@@ -512,7 +512,7 @@ def main(argv):
 if __name__ == "__main__":
     main(sys.argv)
 
-__all__ = [
+__all__ = (
     'yaml_to_workflow',
     'python_to_workflow',
-]
+)

test/api/workflows_format_2/main.py

@@ -36,6 +36,6 @@ def convert_and_import_workflow(has_workflow, **kwds):
         import_kwds["publish"] = True
     return galaxy_interface.import_workflow(workflow, **import_kwds)
 
-__all__ = [
+__all__ = (
     'convert_and_import_workflow',
-]
+)

test/base/driver_util.py

@@ -692,7 +692,7 @@ def drive_test(test_driver_class):
     sys.exit(test_driver_class().run())
 
 
-__all__ = [
+__all__ = (
     "copy_database_template",
     "build_logger",
     "drive_test",
@@ -705,4 +705,4 @@ __all__ = [
     "setup_galaxy_config",
     "TestDriver",
     "wait_for_http_server",
-]
+)

test/functional/tools/multi_data_optional.xml

@@ -0,0 +1,27 @@
+<tool id="multi_data_optional" name="multi_data_optional" version="0.1.0">
+  <command>
+    touch $out1;
+    #for $input in $input1
+    #if $input
+    cat $input >> $out1;
+    #end if
+    #end for
+  </command>
+  <inputs>
+    <param name="input1" type="data" format="txt" multiple="true" label="Data 1" optional="true" />
+  </inputs>
+  <outputs>
+    <data format="txt" name="out1" />
+  </outputs>
+  <tests>
+    <test>
+      <param name="input1" value="simple_line.txt,simple_line_alternative.txt" />
+      <output name="out1">
+        <assert_contents>
+          <has_line line="This is a line of text." />
+          <has_line line="This is a different line of text." />
+        </assert_contents>
+      </output>
+    </test>
+  </tests>
+</tool>

test/functional/tools/samples_tool_conf.xml

@@ -57,6 +57,7 @@
   <tool file="implicit_default_conds.xml" />
   <tool file="multi_data_param.xml" />
   <tool file="multi_data_repeat.xml" />
+  <tool file="multi_data_optional.xml" />
   <tool file="paths_as_file.xml" />
   <tool file="column_param.xml" />
   <tool file="column_multi_param.xml" />

test/unit/tools_support.py

@@ -3,22 +3,22 @@ tool evaluation. Such extensive "fixtures" are something of an anti-pattern
 so use of this should be limitted to tests of very 'extensive' classes.
 """
 
-from collections import defaultdict
 import os.path
-import tempfile
 import shutil
 import string
+import tempfile
+from collections import defaultdict
 
-from galaxy.util.bunch import Bunch
-from galaxy.web.security import SecurityHelper
+import galaxy.datatypes.registry
 import galaxy.model
+from galaxy.jobs import NoopQueue
 from galaxy.model import mapping
 from galaxy.tools import Tool
-from galaxy.util.dbkeys import GenomeBuilds
-from galaxy.jobs import NoopQueue
-from galaxy.tools.parser import get_tool_source
 from galaxy.tools.deps.containers import NullContainerFinder
-import galaxy.datatypes.registry
+from galaxy.tools.parser import get_tool_source
+from galaxy.util.bunch import Bunch
+from galaxy.util.dbkeys import GenomeBuilds
+from galaxy.web.security import SecurityHelper
 
 datatypes_registry = galaxy.datatypes.registry.Registry()
 datatypes_registry.load_datatypes()
@@ -207,4 +207,4 @@ class MockQuery(object):
         return self.class_objects.get(id, None)
 
 
-__all__ = [ UsesApp ]
+__all__ = ( 'UsesApp', )

test/unit/unittest_utils/utility.py

@@ -4,12 +4,7 @@ Unit test utilities.
 import os
 import sys
 import textwrap
-
-from sys import version_info
-if version_info < (2, 7):
-    import unittest2 as unittest
-else:
-    import unittest as unittest
+import unittest
 
 
 # =============================================================================
@@ -45,10 +40,10 @@ def clean_multiline_string( multiline_string, sep='\n' ):
 sys.path[1:1] = [ get_galaxy_libpath(), get_unittest_utils_path() ]
 
 
-__all__ = [
+__all__ = (
     "clean_multiline_string",
     "get_unittest_utils_path",
     "get_galaxy_root",
     "get_galaxy_libpath",
     "unittest",
-]
+)

tools/filters/gff/gff_filter_by_attribute.py

@@ -125,9 +125,9 @@ for i, line in enumerate( open( in_fname ) ):
         %s
         if %s:
             lines_kept += 1
-            print >> out, line
+            print( line, file=out )
     except Exception as e:
-        print e
+        print( e )
         skipped_lines += 1
         if not invalid_line:
             first_invalid_line = i + 1

tools/stats/filtering.py

@@ -219,7 +219,7 @@ for i, line in enumerate( open( in_fname ) ):
 
     if i < num_header_lines:
         lines_kept += 1
-        print >> out, line
+        print( line, file=out )
         continue
 
     if not line or line.startswith( '#' ):
@@ -230,7 +230,7 @@ for i, line in enumerate( open( in_fname ) ):
         %s
         if %s:
             lines_kept += 1
-            print >> out, line
+            print( line, file=out )
     except:
         invalid_lines += 1
         if not invalid_line: