Merge pull request #3003 from jmchilton/docstrings1

Docstring fixes for galaxy.exceptions and galaxy.jobs.metrics.

lib/galaxy/exceptions/__init__.py

@@ -1,14 +1,26 @@
-"""
-Custom exceptions for Galaxy
+"""This module defines Galaxy's custom exceptions.
+
+A Galaxy exception is an exception that extends :class:`MessageException` which
+defines an HTTP status code (represented by the `status_code` attribute) and a
+default error message.
+
+New exceptions should be defined by adding an entry to `error_codes.json` in this
+directory to define a default error message and a Galaxy "error code". A concrete
+Python class should be added in this file defining an HTTP status code (as
+`status_code`) and error code (`error_code`) object loaded dynamically from
+`error_codes.json`.
+
+Reflecting Galaxy's origins as a web application, these exceptions tend to be a
+bit web-oriented. However this module is a dependency of modules and tools that
+have nothing to do with the web - keep this in mind when defining exception names
+and messages.
 """
 
 from ..exceptions import error_codes
 
 
 class MessageException( Exception ):
-    """
-    Exception to make throwing errors from deep in controllers easier.
-    """
+    """Most generic Galaxy exception - indicates merely that some exceptional condition happened."""
     # status code to be set when used with API.
     status_code = 400
     # Error code information embedded into API json responses.

lib/galaxy/exceptions/error_codes.py

@@ -1,3 +1,7 @@
+"""Defines the :class:`ErrorCode` class and instantiates concrete objects from JSON.
+
+See the file error_codes.json for actual error code descriptions.
+"""
 from json import loads
 from pkg_resources import resource_string
 
@@ -8,26 +12,35 @@ from pkg_resources import resource_string
 UNKNOWN_ERROR_MESSAGE = "Unknown error occurred while processing request."
 
 
-class ErrorCode( object ):
+class ErrorCode(object):
+    """Small class allowing object representation for error descriptions loaded from JSON."""
 
-    def __init__( self, code, default_error_message ):
+    def __init__(self, code, default_error_message):
+        """Construct a :class:`ErrorCode` from supplied integer and error message."""
         self.code = code
         self.default_error_message = default_error_message or UNKNOWN_ERROR_MESSAGE
 
-    def __str__( self ):
-        return str( self.default_error_message )
+    def __str__(self):
+        """Return the error code message."""
+        return str(self.default_error_message)
 
-    def __int__( self ):
-        return int( self.code )
+    def __repr__(self):
+        """Return object representation of this error code."""
+        return "ErrorCode[code=%d,message=%s]" % (self.code, str(self.default_error_message))
 
-    @staticmethod
-    def from_dict( entry ):
-        name = entry.get("name")
-        code = entry.get("code")
-        message = entry.get("message")
-        return ( name, ErrorCode( code, message ) )
+    def __int__(self):
+        """Return the error code integer."""
+        return int(self.code)
 
-error_codes_json = resource_string( __name__, 'error_codes.json' ).decode( "UTF-8" )
-for entry in loads( error_codes_json ):
-    name, error_code_obj = ErrorCode.from_dict( entry )
-    globals()[ name ] = error_code_obj
+
+def _from_dict(entry):
+    """Build a :class:`ErrorCode` object from a JSON entry."""
+    name = entry.get("name")
+    code = entry.get("code")
+    message = entry.get("message")
+    return (name, ErrorCode(code, message))
+
+error_codes_json = resource_string(__name__, 'error_codes.json').decode("UTF-8")
+for entry in loads(error_codes_json):
+    name, error_code_obj = _from_dict(entry)
+    globals()[name] = error_code_obj

lib/galaxy/jobs/metrics/__init__.py

@@ -1,3 +1,15 @@
+"""This module defines the job metrics collection framework for Galaxy jobs.
+
+The framework consists of two parts - the :class:`JobMetrics` class and
+individual :class:`JobInstrumenter` plugins.
+
+A :class:`JobMetrics` object reads any number of plugins from a configuration
+source such as an XML file, a YAML file, or a dictionary.
+
+Each :class:`JobInstrumenter` plugin object describes how to inject a bits
+of shell code into a job scripts (before and after tool commands run) and then
+collect the output of these from a job directory.
+"""
 import collections
 import logging
 import os
@@ -7,110 +19,111 @@ from galaxy.util import plugin_config
 
 from ..metrics import formatting
 
-log = logging.getLogger( __name__ )
+log = logging.getLogger(__name__)
 
 
 DEFAULT_FORMATTER = formatting.JobMetricFormatter()
 
 
-class JobMetrics( object ):
+class JobMetrics(object):
+    """Load and store a collection of :class:`JobInstrumenter` objects."""
 
-    def __init__( self, conf_file=None, **kwargs ):
-        """
-        """
+    def __init__(self, conf_file=None, **kwargs):
+        """Load :class:`JobInstrumenter` objects from specified configuration file."""
         self.plugin_classes = self.__plugins_dict()
-        self.default_job_instrumenter = JobInstrumenter.from_file( self.plugin_classes, conf_file, **kwargs )
-        self.job_instrumenters = collections.defaultdict( lambda: self.default_job_instrumenter )
+        self.default_job_instrumenter = JobInstrumenter.from_file(self.plugin_classes, conf_file, **kwargs)
+        self.job_instrumenters = collections.defaultdict(lambda: self.default_job_instrumenter)
 
-    def format( self, plugin, key, value ):
+    def format(self, plugin, key, value):
+        """Find :class:`formatting.JobMetricFormatter` corresponding to instrumented plugin value."""
         if plugin in self.plugin_classes:
             plugin_class = self.plugin_classes[ plugin ]
             formatter = plugin_class.formatter
         else:
             formatter = DEFAULT_FORMATTER
-        return formatter.format( key, value )
+        return formatter.format(key, value)
 
-    def set_destination_conf_file( self, destination_id, conf_file ):
-        instrumenter = JobInstrumenter.from_file( self.plugin_classes, conf_file )
-        self.set_destination_instrumenter( destination_id, instrumenter )
+    def set_destination_conf_file(self, destination_id, conf_file):
+        instrumenter = JobInstrumenter.from_file(self.plugin_classes, conf_file)
+        self.set_destination_instrumenter(destination_id, instrumenter)
 
-    def set_destination_conf_element( self, destination_id, element ):
-        instrumenter = JobInstrumenter( self.plugin_classes, ('xml', element) )
-        self.set_destination_instrumenter( destination_id, instrumenter )
+    def set_destination_conf_element(self, destination_id, element):
+        instrumenter = JobInstrumenter(self.plugin_classes, ('xml', element))
+        self.set_destination_instrumenter(destination_id, instrumenter)
 
-    def set_destination_instrumenter( self, destination_id, job_instrumenter=None ):
+    def set_destination_instrumenter(self, destination_id, job_instrumenter=None):
         if job_instrumenter is None:
             job_instrumenter = NULL_JOB_INSTRUMENTER
         self.job_instrumenters[ destination_id ] = job_instrumenter
 
-    def collect_properties( self, destination_id, job_id, job_directory ):
-        return self.job_instrumenters[ destination_id ].collect_properties( job_id, job_directory )
+    def collect_properties(self, destination_id, job_id, job_directory):
+        return self.job_instrumenters[ destination_id ].collect_properties(job_id, job_directory)
 
-    def __plugins_dict( self ):
+    def __plugins_dict(self):
         import galaxy.jobs.metrics.instrumenters
-        return plugin_config.plugins_dict( galaxy.jobs.metrics.instrumenters, 'plugin_type' )
+        return plugin_config.plugins_dict(galaxy.jobs.metrics.instrumenters, 'plugin_type')
 
 
-class NullJobInstrumenter( object ):
+class NullJobInstrumenter(object):
 
-    def pre_execute_commands( self, job_directory ):
+    def pre_execute_commands(self, job_directory):
         return None
 
-    def post_execute_commands( self, job_directory ):
+    def post_execute_commands(self, job_directory):
         return None
 
-    def collect_properties( self, job_id, job_directory ):
+    def collect_properties(self, job_id, job_directory):
         return {}
 
 NULL_JOB_INSTRUMENTER = NullJobInstrumenter()
 
 
-class JobInstrumenter( object ):
+class JobInstrumenter(object):
 
-    def __init__( self, plugin_classes, plugins_source, **kwargs ):
+    def __init__(self, plugin_classes, plugins_source, **kwargs):
         self.extra_kwargs = kwargs
         self.plugin_classes = plugin_classes
-        self.plugins = self.__plugins_from_source( plugins_source )
+        self.plugins = self.__plugins_from_source(plugins_source)
 
-    def pre_execute_commands( self, job_directory ):
+    def pre_execute_commands(self, job_directory):
         commands = []
         for plugin in self.plugins:
             try:
-                plugin_commands = plugin.pre_execute_instrument( job_directory )
+                plugin_commands = plugin.pre_execute_instrument(job_directory)
                 if plugin_commands:
-                    commands.extend( util.listify( plugin_commands ) )
+                    commands.extend(util.listify(plugin_commands))
             except Exception:
-                log.exception( "Failed to generate pre-execute commands for plugin %s" % plugin )
-        return "\n".join( [ c for c in commands if c ] )
+                log.exception("Failed to generate pre-execute commands for plugin %s" % plugin)
+        return "\n".join([ c for c in commands if c ])
 
-    def post_execute_commands( self, job_directory ):
+    def post_execute_commands(self, job_directory):
         commands = []
         for plugin in self.plugins:
             try:
-                plugin_commands = plugin.post_execute_instrument( job_directory )
+                plugin_commands = plugin.post_execute_instrument(job_directory)
                 if plugin_commands:
-                    commands.extend( util.listify( plugin_commands ) )
+                    commands.extend(util.listify(plugin_commands))
             except Exception:
-                log.exception( "Failed to generate post-execute commands for plugin %s" % plugin )
-        return "\n".join( [ c for c in commands if c ] )
+                log.exception("Failed to generate post-execute commands for plugin %s" % plugin)
+        return "\n".join([ c for c in commands if c ])
 
-    def collect_properties( self, job_id, job_directory ):
+    def collect_properties(self, job_id, job_directory):
         per_plugin_properites = {}
         for plugin in self.plugins:
             try:
-                properties = plugin.job_properties( job_id, job_directory )
+                properties = plugin.job_properties(job_id, job_directory)
                 if properties:
                     per_plugin_properites[ plugin.plugin_type ] = properties
             except Exception:
-                log.exception( "Failed to collect job properties for plugin %s" % plugin )
+                log.exception("Failed to collect job properties for plugin %s" % plugin)
         return per_plugin_properites
 
-    def __plugins_from_source( self, plugins_source ):
+    def __plugins_from_source(self, plugins_source):
         return plugin_config.load_plugins(self.plugin_classes, plugins_source, self.extra_kwargs)
 
     @staticmethod
-    def from_file( plugin_classes, conf_file, **kwargs ):
-        if not conf_file or not os.path.exists( conf_file ):
+    def from_file(plugin_classes, conf_file, **kwargs):
+        if not conf_file or not os.path.exists(conf_file):
             return NULL_JOB_INSTRUMENTER
-        plugins_source = plugin_config.plugin_source_from_path( conf_file )
-        return JobInstrumenter( plugin_classes, plugins_source, **kwargs )
+        plugins_source = plugin_config.plugin_source_from_path(conf_file)
+        return JobInstrumenter(plugin_classes, plugins_source, **kwargs)

lib/galaxy/jobs/metrics/collectl/__init__.py

@@ -1,5 +1,4 @@
-""" This module contains helper functions and data structures for interacting
-with collectl and collectl generated data. More information on collectl can be
-found at: http://collectl.sourceforge.net/.
+"""Helper functions and data structures for interacting with collectl & data.
 
+More information on collectl can be found at: http://collectl.sourceforge.net/.
 """

lib/galaxy/jobs/metrics/collectl/cli.py

@@ -1,3 +1,4 @@
+"""This module describes :class:`CollectlCli` - an abstraction for building collectl command lines."""
 import logging
 import subprocess
 

lib/galaxy/jobs/metrics/collectl/subsystems.py

@@ -1,3 +1,7 @@
+"""Abstractions describing collectl subsystems (specified with the collectl ``-s`` parameter).
+
+Subsystems are essentially monitoring plugins available within collectl.
+"""
 from abc import ABCMeta
 from abc import abstractmethod
 

lib/galaxy/jobs/metrics/formatting.py

@@ -1,18 +1,18 @@
+"""Utilities related to formatting job metrics for human consumption."""
 
 
-class JobMetricFormatter( object ):
-    """ Format job metric key-value pairs for human consumption in Web UI. """
+class JobMetricFormatter(object):
+    """Format job metric key-value pairs for human consumption in Web UI."""
 
-    def format( self, key, value ):
-        return ( str( key ), str( value ) )
+    def format(self, key, value):
+        return (str(key), str(value))
 
 
-# Formatting utilities
-
-def seconds_to_str( value ):
+def seconds_to_str(value):
+    """Convert seconds to a simple simple string describing the amount of time."""
     if value < 60:
         return "%s seconds" % value
     elif value < 3600:
-        return "%s minutes" % ( value / 60 )
+        return "%s minutes" % (value / 60)
     else:
-        return "%s hours and %s minutes" % ( value / 3600, ( value % 3600 ) / 60 )
+        return "%s hours and %s minutes" % (value / 3600, (value % 3600) / 60)

lib/galaxy/jobs/metrics/instrumenters/__init__.py

@@ -1,3 +1,7 @@
+"""This module describes the abstract interface for :class:`InstrumentPlugin`s.
+
+These are responsible for collecting and formatting a coherent set of metrics.
+"""
 import os.path
 
 from abc import ABCMeta
@@ -10,9 +14,7 @@ INSTRUMENT_FILE_PREFIX = "__instrument"
 
 
 class InstrumentPlugin( object ):
-    """ A plugin describing how to instrument Galaxy jobs and retrieve metrics
-    from this instrumentation.
-    """
+    """Describes how to instrument job scripts and retrieve collected metrics."""
     __metaclass__ = ABCMeta
     formatter = formatting.JobMetricFormatter()
 

lib/galaxy/jobs/metrics/instrumenters/collectl.py

@@ -1,3 +1,4 @@
+"""The module describes the ``collectl`` job metrics plugin."""
 import logging
 import os
 import shutil

lib/galaxy/jobs/metrics/instrumenters/core.py

@@ -1,3 +1,4 @@
+"""The module describes the ``core`` job metrics plugin."""
 import logging
 import time
 

lib/galaxy/jobs/metrics/instrumenters/cpuinfo.py

@@ -1,3 +1,4 @@
+"""The module describes the ``cpuinfo`` job metrics plugin."""
 import logging
 import re
 

lib/galaxy/jobs/metrics/instrumenters/env.py

@@ -1,3 +1,4 @@
+"""The module describes the ``env`` job metrics plugin."""
 import logging
 import re
 

lib/galaxy/jobs/metrics/instrumenters/meminfo.py

@@ -1,3 +1,4 @@
+"""The module describes the ``meminfo`` job metrics plugin."""
 import re
 import sys
 

lib/galaxy/jobs/metrics/instrumenters/uname.py

@@ -1,3 +1,4 @@
+"""The module describes the ``uname`` job metrics plugin."""
 from ..instrumenters import InstrumentPlugin
 from ...metrics import formatting