Merge pull request #7992 from jmchilton/more_packages

Add galaxy-tool-util and galaxy-job-metrics packages.

lib/galaxy/app.py

@@ -10,9 +10,8 @@ import galaxy.model.security
 import galaxy.queues
 import galaxy.quota
 import galaxy.security
-from galaxy import config, jobs
+from galaxy import config, job_metrics, jobs
 from galaxy.containers import build_container_interfaces
-from galaxy.jobs import metrics as job_metrics
 from galaxy.managers.collections import DatasetCollectionManager
 from galaxy.managers.folders import FolderManager
 from galaxy.managers.histories import HistoryManager
@@ -20,12 +19,12 @@ from galaxy.managers.libraries import LibraryManager
 from galaxy.managers.tools import DynamicToolManager
 from galaxy.model.tags import GalaxyTagHandler
 from galaxy.queue_worker import GalaxyQueueWorker
+from galaxy.tool_util.deps.views import DependencyResolversView
 from galaxy.tools.cache import (
     ToolCache,
     ToolShedRepositoryCache
 )
 from galaxy.tools.data_manager.manager import DataManagers
-from galaxy.tools.deps.views import DependencyResolversView
 from galaxy.tools.error_reports import ErrorReports
 from galaxy.tools.special_tools import load_lib_tools
 from galaxy.tools.verify import test_data

lib/galaxy/config.py

@@ -25,7 +25,7 @@ from six.moves import configparser
 
 from galaxy.containers import parse_containers_config
 from galaxy.exceptions import ConfigurationError
-from galaxy.tools.deps.container_resolvers.mulled import DEFAULT_CHANNELS
+from galaxy.tool_util.deps.container_resolvers.mulled import DEFAULT_CHANNELS
 from galaxy.util import ExecutionTimer
 from galaxy.util import listify
 from galaxy.util import string_as_bool
@@ -513,7 +513,7 @@ class Configuration(object):
         else:
             self.conda_mapping_files = [
                 self.local_conda_mapping_file,
-                os.path.join(self.root, "lib", "galaxy", "tools", "deps", "resolvers", "default_conda_mapping.yml"),
+                os.path.join(self.root, "lib", "galaxy", "tool_util", "deps", "resolvers", "default_conda_mapping.yml"),
             ]
 
         self.enable_beta_mulled_containers = string_as_bool(kwargs.get('enable_beta_mulled_containers', 'False'))
@@ -1053,8 +1053,8 @@ class ConfiguresGalaxyMixin(object):
     def _configure_toolbox(self):
         from galaxy import tools
         from galaxy.managers.citations import CitationsManager
-        from galaxy.tools.deps import containers
-        from galaxy.tools.deps.dependencies import AppInfo
+        from galaxy.tool_util.deps import containers
+        from galaxy.tool_util.deps.dependencies import AppInfo
         import galaxy.tools.search
 
         self.citations_manager = CitationsManager(self)

lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt

@@ -118,7 +118,7 @@ pbr==5.1.3
 prettytable==0.7.2
 prov==1.5.1
 psutil==5.6.1
-pulsar-galaxy-lib==0.9.1
+pulsar-galaxy-lib==0.11.0
 pyasn1-modules==0.2.4
 pyasn1==0.4.5
 pycparser==2.19

lib/galaxy/job_metrics/__init__.py

@@ -16,7 +16,7 @@ import os
 
 from galaxy import util
 from galaxy.util import plugin_config
-from ..metrics import formatting
+from ..job_metrics import formatting
 
 log = logging.getLogger(__name__)
 
@@ -59,8 +59,8 @@ class JobMetrics(object):
         return self.job_instrumenters[destination_id].collect_properties(job_id, job_directory)
 
     def __plugins_dict(self):
-        import galaxy.jobs.metrics.instrumenters
-        return plugin_config.plugins_dict(galaxy.jobs.metrics.instrumenters, 'plugin_type')
+        import galaxy.job_metrics.instrumenters
+        return plugin_config.plugins_dict(galaxy.job_metrics.instrumenters, 'plugin_type')
 
 
 class NullJobInstrumenter(object):

lib/galaxy/job_metrics/instrumenters/__init__.py

@@ -10,7 +10,7 @@ from abc import (
 
 import six
 
-from ...metrics import formatting
+from .. import formatting
 
 INSTRUMENT_FILE_PREFIX = "__instrument"
 

lib/galaxy/jobs/__init__.py

@@ -32,6 +32,7 @@ from galaxy.jobs.mapper import JobMappingException, JobRunnerMapper
 from galaxy.jobs.runners import BaseJobRunner, JobState
 from galaxy.metadata import get_metadata_compute_strategy
 from galaxy.objectstore import ObjectStorePopulator
+from galaxy.tool_util.deps import requirements
 from galaxy.util import safe_makedirs, unicodify
 from galaxy.util.bunch import Bunch
 from galaxy.util.expressions import ExpressionContext
@@ -551,7 +552,6 @@ class JobConfiguration(ConfiguresHandlers):
         for param in parent.findall('param'):
             key = param.get('id')
             if key in ["container", "container_override"]:
-                from galaxy.tools.deps import requirements
                 containers = map(requirements.container_from_element, param.findall('container'))
                 param_value = list(map(lambda c: c.to_dict(), containers))
             else:

lib/galaxy/jobs/output_checker.py

@@ -1,7 +1,7 @@
 import re
 from logging import getLogger
 
-from galaxy.tools.parser.error_level import StdioErrorLevel
+from galaxy.tool_util.parser.error_level import StdioErrorLevel
 from galaxy.util import unicodify
 from galaxy.util.bunch import Bunch
 

lib/galaxy/jobs/rule_helper.py

@@ -9,7 +9,7 @@ from galaxy import (
     model,
     util
 )
-from galaxy.tools.deps.dependencies import ToolInfo
+from galaxy.tool_util.deps.dependencies import ToolInfo
 
 log = logging.getLogger(__name__)
 

lib/galaxy/jobs/runners/__init__.py

@@ -25,7 +25,7 @@ from galaxy.jobs.runners.util.job_script import (
     job_script,
     write_script
 )
-from galaxy.tools.deps.dependencies import (
+from galaxy.tool_util.deps.dependencies import (
     JobInfo,
     ToolInfo
 )

lib/galaxy/jobs/runners/pulsar.py

@@ -36,7 +36,7 @@ from galaxy.jobs.runners import (
     AsynchronousJobRunner,
     AsynchronousJobState
 )
-from galaxy.tools.deps import dependencies
+from galaxy.tool_util.deps import dependencies
 from galaxy.util import (
     galaxy_directory,
     specs,

lib/galaxy/tool_util/deps/__init__.py

@@ -257,8 +257,8 @@ class DependencyManager(object):
         return plugin_config.load_plugins(self.resolver_classes, plugin_source, extra_kwds)
 
     def __resolvers_dict(self):
-        import galaxy.tools.deps.resolvers
-        return plugin_config.plugins_dict(galaxy.tools.deps.resolvers, 'resolver_type')
+        import galaxy.tool_util.deps.resolvers
+        return plugin_config.plugins_dict(galaxy.tool_util.deps.resolvers, 'resolver_type')
 
 
 class CachedDependencyManager(DependencyManager):

lib/galaxy/tool_util/deps/conda_util.py

@@ -12,7 +12,7 @@ import packaging.version
 import six
 from six.moves import shlex_quote
 
-from galaxy.tools.deps.commands import CommandLineException
+from galaxy.tool_util.deps.commands import CommandLineException
 from galaxy.util import (
     smart_str,
     unicodify

lib/galaxy/tool_util/deps/container_resolvers/__init__.py

@@ -37,7 +37,7 @@ class ContainerResolver(Dictifiable):
     def resolve(self, enabled_container_types, tool_info, **kwds):
         """Find a container matching all supplied requirements for tool.
 
-        The supplied argument is a :class:`galaxy.tools.deps.containers.ToolInfo` description
+        The supplied argument is a :class:`galaxy.tool_util.deps.containers.ToolInfo` description
         of the tool and its requirements.
         """
 

lib/galaxy/tool_util/deps/containers.py

@@ -217,8 +217,8 @@ class ContainerRegistry(object):
         return default_resolvers
 
     def __resolvers_dict(self):
-        import galaxy.tools.deps.container_resolvers
-        return plugin_config.plugins_dict(galaxy.tools.deps.container_resolvers, 'resolver_type')
+        import galaxy.tool_util.deps.container_resolvers
+        return plugin_config.plugins_dict(galaxy.tool_util.deps.container_resolvers, 'resolver_type')
 
     def find_best_container_description(self, enabled_container_types, tool_info):
         """Yield best container description of supplied types matching tool info."""

lib/galaxy/tool_util/deps/dependencies.py

@@ -1,4 +1,4 @@
-from galaxy.tools.deps.requirements import ToolRequirements
+from galaxy.tool_util.deps.requirements import ToolRequirements
 from galaxy.util import bunch
 from .mulled.mulled_build import DEFAULT_CHANNELS
 

lib/galaxy/tool_util/deps/mulled/mulled_build.py

@@ -24,7 +24,7 @@ try:
 except ImportError:
     yaml = None
 
-from galaxy.tools.deps import commands, installable
+from galaxy.tool_util.deps import commands, installable
 from galaxy.util import safe_makedirs
 from ._cli import arg_parser
 from .util import (

lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py

@@ -9,7 +9,7 @@ Build mulled images for requirements defined in a tool:
 
 """
 
-from galaxy.tools.parser import get_tool_source
+from galaxy.tool_util.parser import get_tool_source
 from ._cli import arg_parser
 from .mulled_build import (
     add_build_arguments,

lib/galaxy/tool_util/deps/resolvers/conda.py

@@ -7,8 +7,8 @@ import logging
 import os
 import re
 
-import galaxy.tools.deps.installable
-import galaxy.tools.deps.requirements
+import galaxy.tool_util.deps.installable
+import galaxy.tool_util.deps.requirements
 from . import (
     Dependency,
     DependencyException,
@@ -133,7 +133,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li
         auto_install = _string_as_bool(get_option("auto_install"))
         self.auto_init = _string_as_bool(get_option("auto_init"))
         self.conda_context = conda_context
-        self.disabled = not galaxy.tools.deps.installable.ensure_installed(conda_context, install_conda, self.auto_init)
+        self.disabled = not galaxy.tool_util.deps.installable.ensure_installed(conda_context, install_conda, self.auto_init)
         if self.auto_init and not self.disabled:
             self.conda_context.ensure_conda_build_installed_if_needed()
         self.auto_install = auto_install
@@ -206,7 +206,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li
             if requirement.type != "package":
                 return []
 
-        ToolRequirements = galaxy.tools.deps.requirements.ToolRequirements
+        ToolRequirements = galaxy.tool_util.deps.requirements.ToolRequirements
         expanded_requirements = ToolRequirements([self._expand_requirement(r) for r in requirements])
         if self.versionless:
             conda_targets = [CondaTarget(r.name, version=None) for r in expanded_requirements]

lib/galaxy/tool_util/deps/views.py

@@ -2,7 +2,7 @@ from galaxy import exceptions
 
 
 class DependencyResolversView(object):
-    """ Provide a RESTfulish/JSONy interface to a galaxy.tools.deps.DependencyResolver
+    """ Provide a RESTfulish/JSONy interface to a galaxy.tool_util.deps.DependencyResolver
     object. This can be adapted by the Galaxy web framework or other web apps.
     """
 

lib/galaxy/tool_util/fetcher.py

@@ -7,8 +7,8 @@ class ToolLocationFetcher(object):
         self.resolver_classes = self.__resolvers_dict()
 
     def __resolvers_dict(self):
-        import galaxy.tools.locations
-        return plugin_config.plugins_dict(galaxy.tools.locations, 'scheme')
+        import galaxy.tool_util.locations
+        return plugin_config.plugins_dict(galaxy.tool_util.locations, 'scheme')
 
     def to_tool_path(self, path_or_uri_like, **kwds):
         if "://" not in path_or_uri_like:

lib/galaxy/tool_util/parser/cwl.py

@@ -1,8 +1,8 @@
 import logging
 import os
 
-from galaxy.tools.cwl import tool_proxy
-from galaxy.tools.deps import requirements
+from galaxy.tool_util.cwl import tool_proxy
+from galaxy.tool_util.deps import requirements
 from galaxy.util.odict import odict
 from .error_level import StdioErrorLevel
 from .interface import (

lib/galaxy/tool_util/parser/factory.py

@@ -5,7 +5,7 @@ import logging
 
 import yaml
 
-from galaxy.tools.loader import load_tool_with_refereces
+from galaxy.tool_util.loader import load_tool_with_refereces
 from galaxy.util.odict import odict
 from .cwl import CwlToolSource
 from .interface import InputSource

lib/galaxy/tool_util/parser/xml.py

@@ -7,7 +7,7 @@ from math import isinf
 
 import packaging.version
 
-from galaxy.tools.deps import requirements
+from galaxy.tool_util.deps import requirements
 from galaxy.util import string_as_bool, xml_text, xml_to_string
 from galaxy.util.odict import odict
 from .error_level import StdioErrorLevel

lib/galaxy/tool_util/parser/yaml.py

@@ -1,6 +1,6 @@
 import packaging.version
 
-from galaxy.tools.deps import requirements
+from galaxy.tool_util.deps import requirements
 from galaxy.util.odict import odict
 from .interface import InputSource
 from .interface import PageSource

lib/galaxy/tools/__init__.py

@@ -29,15 +29,26 @@ from galaxy.managers.jobs import JobSearch
 from galaxy.metadata import get_metadata_compute_strategy
 from galaxy.model.tags import GalaxyTagHandler
 from galaxy.queue_worker import send_control_task
+from galaxy.tool_util.deps import (
+    CachedDependencyManager,
+)
+from galaxy.tool_util.fetcher import ToolLocationFetcher
+from galaxy.tool_util.loader import (
+    imported_macro_paths,
+    raw_tool_xml_tree,
+    template_macro_params
+)
+from galaxy.tool_util.parser import (
+    get_tool_source,
+    get_tool_source_from_representation,
+    ToolOutputCollectionPart
+)
+from galaxy.tool_util.parser.xml import XmlPageSource
 from galaxy.tools import expressions
 from galaxy.tools.actions import DefaultToolAction
 from galaxy.tools.actions.data_manager import DataManagerToolAction
 from galaxy.tools.actions.data_source import DataSourceToolAction
 from galaxy.tools.actions.model_operations import ModelOperationToolAction
-from galaxy.tools.deps import (
-    CachedDependencyManager,
-)
-from galaxy.tools.fetcher import ToolLocationFetcher
 from galaxy.tools.parameters import (
     check_param,
     params_from_strings,
@@ -64,12 +75,6 @@ from galaxy.tools.parameters.grouping import Conditional, ConditionalWhen, Repea
 from galaxy.tools.parameters.input_translation import ToolInputTranslator
 from galaxy.tools.parameters.meta import expand_meta_parameters
 from galaxy.tools.parameters.wrapped_json import json_wrap
-from galaxy.tools.parser import (
-    get_tool_source,
-    get_tool_source_from_representation,
-    ToolOutputCollectionPart
-)
-from galaxy.tools.parser.xml import XmlPageSource
 from galaxy.tools.test import parse_tests
 from galaxy.tools.toolbox import BaseGalaxyToolBox
 from galaxy.util import (
@@ -96,11 +101,6 @@ from .execute import (
     execute as execute_job,
     MappingParameters,
 )
-from .loader import (
-    imported_macro_paths,
-    raw_tool_xml_tree,
-    template_macro_params
-)
 from .provided_metadata import parse_tool_provided_metadata
 
 log = logging.getLogger(__name__)

lib/galaxy/tools/actions/__init__.py

@@ -932,7 +932,7 @@ def get_ext_or_implicit_ext(hda):
 
 def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext, python_template_version='3'):
     """ Determines the output format for a dataset based on an abstract
-    description of the output (galaxy.tools.parser.ToolOutput), the parameter
+    description of the output (galaxy.tool_util.parser.ToolOutput), the parameter
     wrappers, a map of the input datasets (name => HDA), and the last input
     extensions in the tool form.
 

lib/galaxy/tools/execute.py

@@ -11,8 +11,8 @@ import six.moves
 
 from galaxy import model
 from galaxy.model.dataset_collections.structure import get_structure, tool_output_to_structure
+from galaxy.tool_util.parser import ToolOutputCollectionPart
 from galaxy.tools.actions import filter_output, on_text_for_names, ToolExecutionCache
-from galaxy.tools.parser import ToolOutputCollectionPart
 from galaxy.util import ExecutionTimer
 
 log = logging.getLogger(__name__)

lib/galaxy/tools/expressions/util.py

@@ -1,4 +1,4 @@
-from galaxy.tools.deps.commands import which
+from galaxy.tool_util.deps.commands import which
 
 
 def find_engine(config):

lib/galaxy/tools/lint.py

@@ -3,9 +3,9 @@ from __future__ import print_function
 
 import inspect
 
+from galaxy.tool_util.parser import get_tool_source
 from galaxy.util import submodules
 from galaxy.util.getargspec import getfullargspec
-from .parser import get_tool_source
 
 
 LEVEL_ALL = "all"

lib/galaxy/tools/loader_directory.py

@@ -8,9 +8,9 @@ import sys
 
 import yaml
 
+from galaxy.tool_util import loader
+from galaxy.tool_util.parser import get_tool_source
 from galaxy.util import checkers
-from .parser import get_tool_source
-from ..tools import loader
 
 log = logging.getLogger(__name__)
 

lib/galaxy/tools/parameters/basic.py

@@ -14,8 +14,8 @@ from six import string_types
 from webob.compat import cgi_FieldStorage
 
 import galaxy.model
-import galaxy.tools.parser
 from galaxy import util
+from galaxy.tool_util.parser import get_input_source as ensure_input_source
 from galaxy.util import (
     sanitize_param,
     string_as_bool,
@@ -35,7 +35,6 @@ from .dataset_matcher import (
     get_dataset_matcher_factory,
 )
 from .sanitize import ToolParameterSanitizer
-from ..parser import get_input_source as ensure_input_source
 
 log = logging.getLogger(__name__)
 

lib/galaxy/tools/parameters/output_collect.py

@@ -20,7 +20,7 @@ from galaxy.model.store.discover import (
     RegexCollectedDatasetMatch,
     UNSET,
 )
-from galaxy.tools.parser.output_collection_def import (
+from galaxy.tool_util.parser.output_collection_def import (
     DEFAULT_DATASET_COLLECTOR_DESCRIPTION,
     INPUT_DBKEY_TOKEN,
     ToolProvidedMetadataDatasetCollection,
@@ -418,7 +418,7 @@ class DatasetCollector(object):
     def __init__(self, dataset_collection_description):
         self.discover_via = dataset_collection_description.discover_via
         # dataset_collection_description is an abstract description
-        # built from the tool parsing module - see galaxy.tools.parser.output_colleciton_def
+        # built from the tool parsing module - see galaxy.tool_util.parser.output_colleciton_def
         self.sort_key = dataset_collection_description.sort_key
         self.sort_reverse = dataset_collection_description.sort_reverse
         self.sort_comp = dataset_collection_description.sort_comp

lib/galaxy/tools/toolbox/base.py

@@ -11,7 +11,7 @@ from six import iteritems
 from six.moves.urllib.parse import urlparse
 
 from galaxy.exceptions import MessageException, ObjectNotFound
-from galaxy.tools.deps import build_dependency_manager
+from galaxy.tool_util.deps import build_dependency_manager
 from galaxy.tools.loader_directory import looks_like_a_tool
 from galaxy.util import (
     ExecutionTimer,

lib/galaxy/tools/verify/interactor.py

@@ -29,7 +29,7 @@ from six import (
 )
 
 from galaxy import util
-from galaxy.tools.parser.interface import TestCollectionDef, TestCollectionOutputDef
+from galaxy.tool_util.parser.interface import TestCollectionDef, TestCollectionOutputDef
 from galaxy.util.bunch import Bunch
 from . import verify
 from .asserts import verify_assertions

lib/galaxy/webapps/galaxy/api/tool_dependencies.py

@@ -3,7 +3,7 @@ API operations allowing clients to manage tool dependencies.
 """
 import logging
 
-from galaxy.tools.deps import views
+from galaxy.tool_util.deps import views
 from galaxy.web import (
     expose_api,
     require_admin

lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py

@@ -7,7 +7,7 @@ from sqlalchemy import or_
 
 import tool_shed.repository_types.util as rt_util
 from galaxy import util, web
-from galaxy.tools.deps import views
+from galaxy.tool_util.deps import views
 from galaxy.util import unicodify
 from galaxy.web.form_builder import CheckboxField
 from tool_shed.galaxy_install import (

lib/galaxy/workflow/extract.py

@@ -4,6 +4,7 @@ histories.
 import logging
 
 from galaxy import exceptions, model
+from galaxy.tool_util.parser import ToolOutputCollectionPart
 from galaxy.tools.parameters.basic import (
     DataCollectionToolParameter,
     DataToolParameter
@@ -13,7 +14,6 @@ from galaxy.tools.parameters.grouping import (
     Repeat,
     Section
 )
-from galaxy.tools.parser import ToolOutputCollectionPart
 from galaxy.util.odict import odict
 from .steps import (
     attach_ordered_steps,