From 17c4af285d75e4a271880b37f4727df59987655a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 4 Oct 2016 09:17:03 -0400 Subject: [PATCH] Docstring fixes for galaxy.exceptions and galaxy.jobs.metrics. xref https://github.com/galaxyproject/galaxy-lib/pull/21. Rebase with spelling fixes thanks to @nsoranzo. --- lib/galaxy/exceptions/__init__.py | 22 +++- lib/galaxy/exceptions/error_codes.py | 45 +++++--- lib/galaxy/jobs/metrics/__init__.py | 103 ++++++++++-------- lib/galaxy/jobs/metrics/collectl/__init__.py | 5 +- lib/galaxy/jobs/metrics/collectl/cli.py | 1 + .../jobs/metrics/collectl/subsystems.py | 4 + lib/galaxy/jobs/metrics/formatting.py | 18 +-- .../jobs/metrics/instrumenters/__init__.py | 8 +- .../jobs/metrics/instrumenters/collectl.py | 1 + lib/galaxy/jobs/metrics/instrumenters/core.py | 1 + .../jobs/metrics/instrumenters/cpuinfo.py | 1 + lib/galaxy/jobs/metrics/instrumenters/env.py | 1 + .../jobs/metrics/instrumenters/meminfo.py | 1 + .../jobs/metrics/instrumenters/uname.py | 1 + 14 files changed, 131 insertions(+), 81 deletions(-) diff --git a/lib/galaxy/exceptions/__init__.py b/lib/galaxy/exceptions/__init__.py index 60e46ee209a..a7fb0220f6c 100644 --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -1,14 +1,26 @@ -""" -Custom exceptions for Galaxy +"""This module defines Galaxy's custom exceptions. + +A Galaxy exception is an exception that extends :class:`MessageException` which +defines an HTTP status code (represented by the `status_code` attribute) and a +default error message. + +New exceptions should be defined by adding an entry to `error_codes.json` in this +directory to define a default error message and a Galaxy "error code". A concrete +Python class should be added in this file defining an HTTP status code (as +`status_code`) and error code (`error_code`) object loaded dynamically from +`error_codes.json`. + +Reflecting Galaxy's origins as a web application, these exceptions tend to be a +bit web-oriented. However this module is a dependency of modules and tools that +have nothing to do with the web - keep this in mind when defining exception names +and messages. """ from ..exceptions import error_codes class MessageException( Exception ): - """ - Exception to make throwing errors from deep in controllers easier. - """ + """Most generic Galaxy exception - indicates merely that some exceptional condition happened.""" # status code to be set when used with API. status_code = 400 # Error code information embedded into API json responses. diff --git a/lib/galaxy/exceptions/error_codes.py b/lib/galaxy/exceptions/error_codes.py index e921d154905..e41f0828313 100644 --- a/lib/galaxy/exceptions/error_codes.py +++ b/lib/galaxy/exceptions/error_codes.py @@ -1,3 +1,7 @@ +"""Defines the :class:`ErrorCode` class and instantiates concrete objects from JSON. + +See the file error_codes.json for actual error code descriptions. +""" from json import loads from pkg_resources import resource_string @@ -8,26 +12,35 @@ from pkg_resources import resource_string UNKNOWN_ERROR_MESSAGE = "Unknown error occurred while processing request." -class ErrorCode( object ): +class ErrorCode(object): + """Small class allowing object representation for error descriptions loaded from JSON.""" - def __init__( self, code, default_error_message ): + def __init__(self, code, default_error_message): + """Construct a :class:`ErrorCode` from supplied integer and error message.""" self.code = code self.default_error_message = default_error_message or UNKNOWN_ERROR_MESSAGE - def __str__( self ): - return str( self.default_error_message ) + def __str__(self): + """Return the error code message.""" + return str(self.default_error_message) - def __int__( self ): - return int( self.code ) + def __repr__(self): + """Return object representation of this error code.""" + return "ErrorCode[code=%d,message=%s]" % (self.code, str(self.default_error_message)) - @staticmethod - def from_dict( entry ): - name = entry.get("name") - code = entry.get("code") - message = entry.get("message") - return ( name, ErrorCode( code, message ) ) + def __int__(self): + """Return the error code integer.""" + return int(self.code) -error_codes_json = resource_string( __name__, 'error_codes.json' ).decode( "UTF-8" ) -for entry in loads( error_codes_json ): - name, error_code_obj = ErrorCode.from_dict( entry ) - globals()[ name ] = error_code_obj + +def _from_dict(entry): + """Build a :class:`ErrorCode` object from a JSON entry.""" + name = entry.get("name") + code = entry.get("code") + message = entry.get("message") + return (name, ErrorCode(code, message)) + +error_codes_json = resource_string(__name__, 'error_codes.json').decode("UTF-8") +for entry in loads(error_codes_json): + name, error_code_obj = _from_dict(entry) + globals()[name] = error_code_obj diff --git a/lib/galaxy/jobs/metrics/__init__.py b/lib/galaxy/jobs/metrics/__init__.py index d3f93576ef8..87c5fafa54f 100644 --- a/lib/galaxy/jobs/metrics/__init__.py +++ b/lib/galaxy/jobs/metrics/__init__.py @@ -1,3 +1,15 @@ +"""This module defines the job metrics collection framework for Galaxy jobs. + +The framework consists of two parts - the :class:`JobMetrics` class and +individual :class:`JobInstrumenter` plugins. + +A :class:`JobMetrics` object reads any number of plugins from a configuration +source such as an XML file, a YAML file, or a dictionary. + +Each :class:`JobInstrumenter` plugin object describes how to inject a bits +of shell code into a job scripts (before and after tool commands run) and then +collect the output of these from a job directory. +""" import collections import logging import os @@ -7,110 +19,111 @@ from galaxy.util import plugin_config from ..metrics import formatting -log = logging.getLogger( __name__ ) +log = logging.getLogger(__name__) DEFAULT_FORMATTER = formatting.JobMetricFormatter() -class JobMetrics( object ): +class JobMetrics(object): + """Load and store a collection of :class:`JobInstrumenter` objects.""" - def __init__( self, conf_file=None, **kwargs ): - """ - """ + def __init__(self, conf_file=None, **kwargs): + """Load :class:`JobInstrumenter` objects from specified configuration file.""" self.plugin_classes = self.__plugins_dict() - self.default_job_instrumenter = JobInstrumenter.from_file( self.plugin_classes, conf_file, **kwargs ) - self.job_instrumenters = collections.defaultdict( lambda: self.default_job_instrumenter ) + self.default_job_instrumenter = JobInstrumenter.from_file(self.plugin_classes, conf_file, **kwargs) + self.job_instrumenters = collections.defaultdict(lambda: self.default_job_instrumenter) - def format( self, plugin, key, value ): + def format(self, plugin, key, value): + """Find :class:`formatting.JobMetricFormatter` corresponding to instrumented plugin value.""" if plugin in self.plugin_classes: plugin_class = self.plugin_classes[ plugin ] formatter = plugin_class.formatter else: formatter = DEFAULT_FORMATTER - return formatter.format( key, value ) + return formatter.format(key, value) - def set_destination_conf_file( self, destination_id, conf_file ): - instrumenter = JobInstrumenter.from_file( self.plugin_classes, conf_file ) - self.set_destination_instrumenter( destination_id, instrumenter ) + def set_destination_conf_file(self, destination_id, conf_file): + instrumenter = JobInstrumenter.from_file(self.plugin_classes, conf_file) + self.set_destination_instrumenter(destination_id, instrumenter) - def set_destination_conf_element( self, destination_id, element ): - instrumenter = JobInstrumenter( self.plugin_classes, ('xml', element) ) - self.set_destination_instrumenter( destination_id, instrumenter ) + def set_destination_conf_element(self, destination_id, element): + instrumenter = JobInstrumenter(self.plugin_classes, ('xml', element)) + self.set_destination_instrumenter(destination_id, instrumenter) - def set_destination_instrumenter( self, destination_id, job_instrumenter=None ): + def set_destination_instrumenter(self, destination_id, job_instrumenter=None): if job_instrumenter is None: job_instrumenter = NULL_JOB_INSTRUMENTER self.job_instrumenters[ destination_id ] = job_instrumenter - def collect_properties( self, destination_id, job_id, job_directory ): - return self.job_instrumenters[ destination_id ].collect_properties( job_id, job_directory ) + def collect_properties(self, destination_id, job_id, job_directory): + return self.job_instrumenters[ destination_id ].collect_properties(job_id, job_directory) - def __plugins_dict( self ): + def __plugins_dict(self): import galaxy.jobs.metrics.instrumenters - return plugin_config.plugins_dict( galaxy.jobs.metrics.instrumenters, 'plugin_type' ) + return plugin_config.plugins_dict(galaxy.jobs.metrics.instrumenters, 'plugin_type') -class NullJobInstrumenter( object ): +class NullJobInstrumenter(object): - def pre_execute_commands( self, job_directory ): + def pre_execute_commands(self, job_directory): return None - def post_execute_commands( self, job_directory ): + def post_execute_commands(self, job_directory): return None - def collect_properties( self, job_id, job_directory ): + def collect_properties(self, job_id, job_directory): return {} NULL_JOB_INSTRUMENTER = NullJobInstrumenter() -class JobInstrumenter( object ): +class JobInstrumenter(object): - def __init__( self, plugin_classes, plugins_source, **kwargs ): + def __init__(self, plugin_classes, plugins_source, **kwargs): self.extra_kwargs = kwargs self.plugin_classes = plugin_classes - self.plugins = self.__plugins_from_source( plugins_source ) + self.plugins = self.__plugins_from_source(plugins_source) - def pre_execute_commands( self, job_directory ): + def pre_execute_commands(self, job_directory): commands = [] for plugin in self.plugins: try: - plugin_commands = plugin.pre_execute_instrument( job_directory ) + plugin_commands = plugin.pre_execute_instrument(job_directory) if plugin_commands: - commands.extend( util.listify( plugin_commands ) ) + commands.extend(util.listify(plugin_commands)) except Exception: - log.exception( "Failed to generate pre-execute commands for plugin %s" % plugin ) - return "\n".join( [ c for c in commands if c ] ) + log.exception("Failed to generate pre-execute commands for plugin %s" % plugin) + return "\n".join([ c for c in commands if c ]) - def post_execute_commands( self, job_directory ): + def post_execute_commands(self, job_directory): commands = [] for plugin in self.plugins: try: - plugin_commands = plugin.post_execute_instrument( job_directory ) + plugin_commands = plugin.post_execute_instrument(job_directory) if plugin_commands: - commands.extend( util.listify( plugin_commands ) ) + commands.extend(util.listify(plugin_commands)) except Exception: - log.exception( "Failed to generate post-execute commands for plugin %s" % plugin ) - return "\n".join( [ c for c in commands if c ] ) + log.exception("Failed to generate post-execute commands for plugin %s" % plugin) + return "\n".join([ c for c in commands if c ]) - def collect_properties( self, job_id, job_directory ): + def collect_properties(self, job_id, job_directory): per_plugin_properites = {} for plugin in self.plugins: try: - properties = plugin.job_properties( job_id, job_directory ) + properties = plugin.job_properties(job_id, job_directory) if properties: per_plugin_properites[ plugin.plugin_type ] = properties except Exception: - log.exception( "Failed to collect job properties for plugin %s" % plugin ) + log.exception("Failed to collect job properties for plugin %s" % plugin) return per_plugin_properites - def __plugins_from_source( self, plugins_source ): + def __plugins_from_source(self, plugins_source): return plugin_config.load_plugins(self.plugin_classes, plugins_source, self.extra_kwargs) @staticmethod - def from_file( plugin_classes, conf_file, **kwargs ): - if not conf_file or not os.path.exists( conf_file ): + def from_file(plugin_classes, conf_file, **kwargs): + if not conf_file or not os.path.exists(conf_file): return NULL_JOB_INSTRUMENTER - plugins_source = plugin_config.plugin_source_from_path( conf_file ) - return JobInstrumenter( plugin_classes, plugins_source, **kwargs ) + plugins_source = plugin_config.plugin_source_from_path(conf_file) + return JobInstrumenter(plugin_classes, plugins_source, **kwargs) diff --git a/lib/galaxy/jobs/metrics/collectl/__init__.py b/lib/galaxy/jobs/metrics/collectl/__init__.py index 3445fafd638..c3e8815049d 100644 --- a/lib/galaxy/jobs/metrics/collectl/__init__.py +++ b/lib/galaxy/jobs/metrics/collectl/__init__.py @@ -1,5 +1,4 @@ -""" This module contains helper functions and data structures for interacting -with collectl and collectl generated data. More information on collectl can be -found at: http://collectl.sourceforge.net/. +"""Helper functions and data structures for interacting with collectl & data. +More information on collectl can be found at: http://collectl.sourceforge.net/. """ diff --git a/lib/galaxy/jobs/metrics/collectl/cli.py b/lib/galaxy/jobs/metrics/collectl/cli.py index 413e5b609be..c2738d98863 100644 --- a/lib/galaxy/jobs/metrics/collectl/cli.py +++ b/lib/galaxy/jobs/metrics/collectl/cli.py @@ -1,3 +1,4 @@ +"""This module describes :class:`CollectlCli` - an abstraction for building collectl command lines.""" import logging import subprocess diff --git a/lib/galaxy/jobs/metrics/collectl/subsystems.py b/lib/galaxy/jobs/metrics/collectl/subsystems.py index 28e2bece559..9fa192c79ee 100644 --- a/lib/galaxy/jobs/metrics/collectl/subsystems.py +++ b/lib/galaxy/jobs/metrics/collectl/subsystems.py @@ -1,3 +1,7 @@ +"""Abstractions describing collectl subsystems (specified with the collectl ``-s`` parameter). + +Subsystems are essentially monitoring plugins available within collectl. +""" from abc import ABCMeta from abc import abstractmethod diff --git a/lib/galaxy/jobs/metrics/formatting.py b/lib/galaxy/jobs/metrics/formatting.py index 1147edd5e73..6bda18418c2 100644 --- a/lib/galaxy/jobs/metrics/formatting.py +++ b/lib/galaxy/jobs/metrics/formatting.py @@ -1,18 +1,18 @@ +"""Utilities related to formatting job metrics for human consumption.""" -class JobMetricFormatter( object ): - """ Format job metric key-value pairs for human consumption in Web UI. """ +class JobMetricFormatter(object): + """Format job metric key-value pairs for human consumption in Web UI.""" - def format( self, key, value ): - return ( str( key ), str( value ) ) + def format(self, key, value): + return (str(key), str(value)) -# Formatting utilities - -def seconds_to_str( value ): +def seconds_to_str(value): + """Convert seconds to a simple simple string describing the amount of time.""" if value < 60: return "%s seconds" % value elif value < 3600: - return "%s minutes" % ( value / 60 ) + return "%s minutes" % (value / 60) else: - return "%s hours and %s minutes" % ( value / 3600, ( value % 3600 ) / 60 ) + return "%s hours and %s minutes" % (value / 3600, (value % 3600) / 60) diff --git a/lib/galaxy/jobs/metrics/instrumenters/__init__.py b/lib/galaxy/jobs/metrics/instrumenters/__init__.py index e6a6e1ed6d6..0a81cffb56b 100644 --- a/lib/galaxy/jobs/metrics/instrumenters/__init__.py +++ b/lib/galaxy/jobs/metrics/instrumenters/__init__.py @@ -1,3 +1,7 @@ +"""This module describes the abstract interface for :class:`InstrumentPlugin`s. + +These are responsible for collecting and formatting a coherent set of metrics. +""" import os.path from abc import ABCMeta @@ -10,9 +14,7 @@ INSTRUMENT_FILE_PREFIX = "__instrument" class InstrumentPlugin( object ): - """ A plugin describing how to instrument Galaxy jobs and retrieve metrics - from this instrumentation. - """ + """Describes how to instrument job scripts and retrieve collected metrics.""" __metaclass__ = ABCMeta formatter = formatting.JobMetricFormatter() diff --git a/lib/galaxy/jobs/metrics/instrumenters/collectl.py b/lib/galaxy/jobs/metrics/instrumenters/collectl.py index ee0358be390..e5a117a0083 100644 --- a/lib/galaxy/jobs/metrics/instrumenters/collectl.py +++ b/lib/galaxy/jobs/metrics/instrumenters/collectl.py @@ -1,3 +1,4 @@ +"""The module describes the ``collectl`` job metrics plugin.""" import logging import os import shutil diff --git a/lib/galaxy/jobs/metrics/instrumenters/core.py b/lib/galaxy/jobs/metrics/instrumenters/core.py index 989cbd81c94..628ada70eec 100644 --- a/lib/galaxy/jobs/metrics/instrumenters/core.py +++ b/lib/galaxy/jobs/metrics/instrumenters/core.py @@ -1,3 +1,4 @@ +"""The module describes the ``core`` job metrics plugin.""" import logging import time diff --git a/lib/galaxy/jobs/metrics/instrumenters/cpuinfo.py b/lib/galaxy/jobs/metrics/instrumenters/cpuinfo.py index 0f4fd93fe85..1cd5c228cec 100644 --- a/lib/galaxy/jobs/metrics/instrumenters/cpuinfo.py +++ b/lib/galaxy/jobs/metrics/instrumenters/cpuinfo.py @@ -1,3 +1,4 @@ +"""The module describes the ``cpuinfo`` job metrics plugin.""" import logging import re diff --git a/lib/galaxy/jobs/metrics/instrumenters/env.py b/lib/galaxy/jobs/metrics/instrumenters/env.py index d6dc1b554fd..dd8420b7085 100644 --- a/lib/galaxy/jobs/metrics/instrumenters/env.py +++ b/lib/galaxy/jobs/metrics/instrumenters/env.py @@ -1,3 +1,4 @@ +"""The module describes the ``env`` job metrics plugin.""" import logging import re diff --git a/lib/galaxy/jobs/metrics/instrumenters/meminfo.py b/lib/galaxy/jobs/metrics/instrumenters/meminfo.py index 20125620d00..1c13bbcc97f 100644 --- a/lib/galaxy/jobs/metrics/instrumenters/meminfo.py +++ b/lib/galaxy/jobs/metrics/instrumenters/meminfo.py @@ -1,3 +1,4 @@ +"""The module describes the ``meminfo`` job metrics plugin.""" import re import sys diff --git a/lib/galaxy/jobs/metrics/instrumenters/uname.py b/lib/galaxy/jobs/metrics/instrumenters/uname.py index 0744ffef282..2ad9571672e 100644 --- a/lib/galaxy/jobs/metrics/instrumenters/uname.py +++ b/lib/galaxy/jobs/metrics/instrumenters/uname.py @@ -1,3 +1,4 @@ +"""The module describes the ``uname`` job metrics plugin.""" from ..instrumenters import InstrumentPlugin from ...metrics import formatting