Merge pull request #3086 from nsoranzo/fix_doc

Properly integrate tool XML schema with docs
2026-09-24 16:30:27 +08:00 · 2016-10-31 10:48:19 -04:00
parent f0c5d5ca39 527e332aa3
commit f9ee0f8637
60 changed files with 1348 additions and 1395 deletions
@@ -2,7 +2,6 @@
 .tox/
 client/node_modules/
 database/
-doc/patch.py
 doc/source/conf.py
 eggs/
 lib/galaxy/util/jstree.py
@@ -5,6 +5,7 @@ cron/cleanup_datasets.py
 cron/parse_builds_3_sites.py
 cron/parse_builds.py
 doc/parse_gx_xsd.py
+doc/patch.py
 lib/galaxy/actions/
 lib/galaxy/auth/
 lib/galaxy/config.py
@@ -2,6 +2,7 @@ client/galaxy/style/source_material/circle.py
 contrib/
 cron/
 doc/parse_gx_xsd.py
+doc/patch.py
 lib/galaxy/actions/
 lib/galaxy/auth/
 lib/galaxy/config.py
@@ -127,8 +127,8 @@ bower_components

 # Documentation build files.
 doc/build
-doc/schema.html
 doc/schema.md
+doc/source/dev/schema.rst

 # Misc
 *.orig
@@ -34,15 +34,6 @@ docs-slides-ready:
 docs-slides-export: docs-slides-ready
 	$(SLIDESHOW_TO_PDF) $(SLIDESHOW_DIR)/galaxy_architecture/galaxy_architecture.html

-docs-schema-ready: ## Build Github-flavored Markdown from Galaxy Tool XSD (expects libxml in environment)
-	python $(DOCS_DIR)/parse_gx_xsd.py > $(DOCS_DIR)/schema.md
-
-docs-schema-html: docs-schema-ready ## Convert Galaxy Tool XSD Markdown docs into HTML (expects pandoc in environment)
-	pandoc $(DOCS_DIR)/schema.md -f markdown_github -s -o $(DOCS_DIR)/schema.html
-
-open-docs-schema: docs-schema-html ## Open HTML generated from Galaxy Tool XSD.
-	$(OPEN_RESOURCE) $(DOCS_DIR)/schema.html
-
 _open-docs:
 	$(OPEN_RESOURCE) $(DOCS_DIR)/_build/html/index.html

@@ -17,8 +17,9 @@ ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) sou
 # the i18n builder cannot share the environment and doctrees with the others
 I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source

-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext updaterst
+GENERATED_RST = source/dev/schema.rst

+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext updaterst

 help:
 	@echo "Please use \`make <target>' where <target> is one of"
@@ -43,44 +44,49 @@ help:
 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
 	@echo "  updaterst  to update sphinx rst to reflect code structure changes"

+schema.md: parse_gx_xsd.py schema_template.md ../lib/galaxy/tools/xsd/galaxy.xsd ## Build Github-flavored Markdown from Galaxy Tool XSD (expects libxml in environment)
+	python parse_gx_xsd.py schema_template.md ../lib/galaxy/tools/xsd/galaxy.xsd > $@
+
+source/dev/schema.rst: schema.md ## Convert Galaxy Tool XSD Markdown docs into reStructuredText (expects pandoc in environment)
+	pandoc schema.md -f markdown_github-hard_line_breaks -s --toc --toc-depth=2 -o $@

 # might also want to do
 # cd source/lib; hg revert; rm *.rst.orig;  or not.
 clean:
-	-rm -rf $(BUILDDIR)/* $(UPDATEWORKDIR)
+	-rm -rf $(BUILDDIR)/* $(UPDATEWORKDIR) schema.md $(GENERATED_RST)

-html: $(TOOLDATABUILDFILES)
+html: $(GENERATED_RST)
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."

-dirhtml:
+dirhtml: $(GENERATED_RST)
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."

-singlehtml:
+singlehtml: $(GENERATED_RST)
 	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
 	@echo
 	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."

-pickle:
+pickle: $(GENERATED_RST)
 	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
 	@echo
 	@echo "Build finished; now you can process the pickle files."

-json:
+json: $(GENERATED_RST)
 	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
 	@echo
 	@echo "Build finished; now you can process the JSON files."

-htmlhelp:
+htmlhelp: $(GENERATED_RST)
 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
 	@echo
 	@echo "Build finished; now you can run HTML Help Workshop with the" \
 	      ".hhp project file in $(BUILDDIR)/htmlhelp."

-qthelp:
+qthelp: $(GENERATED_RST)
 	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
 	@echo
 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
@@ -89,7 +95,7 @@ qthelp:
 	@echo "To view the help file:"
 	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Galaxy.qhc"

-devhelp:
+devhelp: $(GENERATED_RST)
 	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
 	@echo
 	@echo "Build finished."
@@ -98,30 +104,30 @@ devhelp:
 	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Galaxy"
 	@echo "# devhelp"

-epub:
+epub: $(GENERATED_RST)
 	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
 	@echo
 	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."

-latex:
+latex: $(GENERATED_RST)
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo
 	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
 	@echo "Run \`make' in that directory to run these through (pdf)latex" \
 	      "(use \`make latexpdf' here to do that automatically)."

-latexpdf:
+latexpdf: $(GENERATED_RST)
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 	@echo "Running LaTeX files through pdflatex..."
 	$(MAKE) -C $(BUILDDIR)/latex all-pdf
 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."

-text:
+text: $(GENERATED_RST)
 	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
 	@echo
 	@echo "Build finished. The text files are in $(BUILDDIR)/text."

-man:
+man: $(GENERATED_RST)
 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
 	@echo
 	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
@@ -133,29 +139,29 @@ texinfo:
 	@echo "Run \`make' in that directory to run these through makeinfo" \
 	      "(use \`make info' here to do that automatically)."

-info:
+info: $(GENERATED_RST)
 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
 	@echo "Running Texinfo files through makeinfo..."
 	make -C $(BUILDDIR)/texinfo info
 	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."

-gettext:
+gettext: $(GENERATED_RST)
 	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
 	@echo
 	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."

-changes:
+changes: $(GENERATED_RST)
 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
 	@echo
 	@echo "The overview file is in $(BUILDDIR)/changes."

-linkcheck:
+linkcheck: $(GENERATED_RST)
 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
 	@echo
 	@echo "Link check complete; look for any errors in the above output " \
 	      "or in $(BUILDDIR)/linkcheck/output.txt."

-doctest:
+doctest: $(GENERATED_RST)
 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
 	@echo "Testing of doctests in the sources finished, look at the " \
 	      "results in $(BUILDDIR)/doctest/output.txt."
@@ -2,17 +2,19 @@
 # TODO: Add examples, tables and best practice links to command
 # TODO: Examples of truevalue, falsevalue
 # TODO: Test param extra_file
-# Things dropped from TOC (still documented inside schema).
+# Things dropped from schema_template.md (still documented inside schema).
 #  - request_parameter_translation
 from __future__ import print_function

+import sys
+
 from lxml import etree
 from six import StringIO

-with open("doc/schema_template.md", "r") as f:
+with open(sys.argv[1], "r") as f:
    MARKDOWN_TEMPLATE = f.read()

-with open("lib/galaxy/tools/xsd/galaxy.xsd", "r") as f:
+with open(sys.argv[2], "r") as f:
    xmlschema_doc = etree.parse(f)

 markdown_buffer = StringIO()
@@ -23,42 +25,10 @@ def main():
    for line in MARKDOWN_TEMPLATE.splitlines():
        if line.startswith("$tag:"):
            print(Tag(line).build_help())
-        elif line.startswith("$toc"):
-            print_toc()
        else:
            print(line)


-def print_toc():
-    tags = []
-    for line in MARKDOWN_TEMPLATE.splitlines():
-        if line.startswith("$tag:"):
-            tags.append(Tag(line))
-
-    for i, tag in enumerate(tags):
-        try:
-            next_tag = tags[i + 1]
-            next_count = next_tag.title.count("|")
-        except IndexError:
-            next_tag = None
-            next_count = None
-        line = ""
-        count = tag.title.count("|")
-        for c in range(count):
-            if next_count is not None and c < next_count:
-                if c == 0:
-                    line += "├──"
-                else:
-                    line += "┼──"
-            else:
-                if c == 0:
-                    line += "└──"
-                else:
-                    line += "┴──"
-        line += "[``<" + tag.title.split("|")[-1] + ">``](#" + tag.title + ")  "
-        print(line)
-
-
 class Tag(object):

    def __init__(self, line):
@@ -84,7 +54,6 @@ class Tag(object):

        title = self.title
        tag_help = StringIO()
-        tag_help.write("""\n<a name="%s"></a>\n""" % title)
        tag_help.write("## " + " > ".join(["``%s``" % p for p in title.split("|")]))
        tag_help.write("\n")
        tag_help.write(_build_tag(tag, self.hide_attributes))
@@ -100,9 +69,10 @@ def _build_tag(tag, hide_attributes):
    text = annotation_el.find("{http://www.w3.org/2001/XMLSchema}documentation").text
    for line in text.splitlines():
        if line.startswith("$attribute_list:"):
-            attributes_str = line.split(":", 1)[1]
+            attributes_str, header_level = line.split(":")[1:3]
            attribute_names = attributes_str.split(",")
-            text = text.replace(line, _build_attributes_table(tag, attributes, attribute_names=attribute_names))
+            header_level = int(header_level)
+            text = text.replace(line, _build_attributes_table(tag, attributes, attribute_names=attribute_names, header_level=header_level))
        if line.startswith("$assertions"):
            assertions_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}complexType[@name='TestAssertions']")
            assertion_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}group[@name='TestAssertion']")
@@ -132,15 +102,16 @@ def _get_bp_link(annotation_el):
    anchor = annotation_el.attrib.get("{http://galaxyproject.org/xml/1.0}best_practices", None)
    link = None
    if anchor:
-        link = "http://planemo.readthedocs.io/en/latest/standards/docs/best_practices/tool_xml.html#%s" % anchor
+        link = "https://planemo.readthedocs.io/en/latest/standards/docs/best_practices/tool_xml.html#%s" % anchor
    return link


-def _build_attributes_table(tag, attributes, hide_attributes=False, attribute_names=None):
+def _build_attributes_table(tag, attributes, hide_attributes=False, attribute_names=None, header_level=3):
    attribute_table = StringIO()
    attribute_table.write("\n\n")
    if attributes and not hide_attributes:
-        attribute_table.write("\n### Attributes\n")
+        header_prefix = '#' * header_level
+        attribute_table.write("\n%s Attributes\n" % header_prefix)
        attribute_table.write("Attribute | Details | Required\n")
        attribute_table.write("--- | --- | ---\n")
        for attribute in attributes:
@@ -10,11 +10,9 @@ If you find a bug please report it [here](https://github.com/galaxyproject/galax

 This document serves as reference documentation. If you would like to learn
 how to build tools for Galaxy,
-[Planemo](http://planemo.readthedocs.io/en/latest/writing.html) features a
+[Planemo](https://planemo.readthedocs.io/en/latest/writing.html) features a
 number of tutorials on building Galaxy tools that would better serve that purpose.

-$toc
-
 $tag:tool://element[@name='tool']
 $tag:tool|description://element[@name='tool']//element[@name='description']
 $tag:tool|version_command://complexType[@name='VersionCommand']
@@ -1,6 +1,6 @@
-=================================
+===========================
 Conda for Tool Dependencies
-=================================
+===========================

 Galaxy tools (also called wrappers) traditionally use Tool Shed package
 recipes to install their dependencies. At the tool's installation time
@@ -46,7 +46,7 @@ Below we answer some common questions (collected by Lance Parsons):


 1. How do I enable Conda dependency resolution for Galaxy jobs?
-*********************************************************************************
+***************************************************************

 Galaxy's dependency job resolution is managed via
 ``dependency_resolvers_conf.xml`` configuration file. Most Galaxy administrators
@@ -103,7 +103,7 @@ See `galaxy.ini.sample`_ for the complete list.


 2. How do Conda dependencies work? Where do things get installed?
-*********************************************************************************
+*****************************************************************

 In contrast to the TS dependency system, which was used exclusively by Galaxy,
 Conda is a pre-existing, independent project. With Conda, it is possible for an
@@ -154,7 +154,7 @@ be configured in the ``dependency_resolvers_conf.xml`` and the ``galaxy.ini`` fi


 3. What is required to make use of this? Any specific packages, Galaxy revision, OS version, etc.?
-*********************************************************************************
+**************************************************************************************************

 The minimum required version of Galaxy to use Conda is 16.01, however
 version 16.07 or greater is recommended. The 16.07 release of Galaxy has
@@ -169,7 +169,7 @@ systems newer than 2007.


 4. If I have Conda enabled, what do I need to do to install tools using it? For example, how can I install the latest Trinity? And how will I know the dependencies are installed?
-*********************************************************************************
+**********************************************************************************************************************************************************************************

 This depends on your ``galaxy.ini`` setting. Starting with release 16.07, Galaxy
 can automatically install the Conda package manager for you if you have enabled
@@ -198,7 +198,7 @@ are no available TS dependencies.


 5. Can I mix traditional Galaxy packages and Conda packages?
-*********************************************************************************
+************************************************************

 Yes, the way this works is that Galaxy goes through the list of
 requirements for a tool, and then determines for each requirement if it
@@ -216,7 +216,7 @@ The first system that satisfies a requirement will be used. See


 6. How do I know what system is being used by a given tool?
-*********************************************************************************
+***********************************************************

 The Galaxy log will show which dependency resolution system is used
 to satisfy each tool dependency and you can specify priorities using the
@@ -226,7 +226,7 @@ Admin panel.


 7. How do I go about specifying Conda dependencies for a tool? All the docs still seem to recommend (or exclusively discuss) the ``tool_dependencies.xml`` method.
-*********************************************************************************
+******************************************************************************************************************************************************************

 The simple answer is: you don't need to do much to make Conda work for a tool.

@@ -243,7 +243,7 @@ deprecate it everywhere.


 8. During tool installation what if there is no Conda package available for a given requirement? What if the requirement is resolved in a different software than the original wrapper author meant to use?
-*********************************************************************************
+***********************************************************************************************************************************************************************************************************

 If there is no Conda package available during tool installation the tool
 will install automatically, and can be used if its dependencies are
@@ -256,7 +256,7 @@ installed from the Tool Shed.


 9. Where can I find a list of existing Conda packages that I can point to, so I don't have to reinvent the wheel for common dependencies?
-*********************************************************************************
+*****************************************************************************************************************************************

 With Conda package manager installed on your system, run:

@@ -270,7 +270,7 @@ Galaxy. If you find your package, you are ready to go. If not please


 10. How can I create a new Conda package for a dependency?
-*********************************************************************************
+**********************************************************

 Adding a package to the BioConda or IUC Conda channels will make it
 available for Galaxy tools to use as a dependency. To learn how, get in
@@ -284,7 +284,7 @@ pypi, cran, or cpan for you (mostly) automatically.


 11. Is there a way to convert traditional Tool Shed package recipes that are not yet in a Conda channel?
-*********************************************************************************
+********************************************************************************************************

 First, you do not need to do anything to your wrapper as long as the
 package name in the requirement tag matches the name of correct
@@ -297,7 +297,7 @@ leave the old versions as they are – simply because of time.


 12. What is the recommendation for existing installations? Will I continue to maintain both systems or migrate to the new Conda system eventually?
-*********************************************************************************
+**************************************************************************************************************************************************

 Old tools will use the traditional installation system; this system will
 stay and will be supported for installing old tools to guarantee sustainability
@@ -305,7 +305,7 @@ and reproducibility. New tools from the IUC, may be Conda only.


 13. What can I do if Conda doesn't work for me?
-*********************************************************************************
+***********************************************

 There is currently a limitation in the way Conda packages are being
 built. This limitation will be addressed shortly by the Conda community,
@@ -61,7 +61,7 @@ Once Node and npm are ready to go, you'll need to install the dependencies

 Running ``node lib/main.js --help`` should produce some useful help text

-.. code-block::
+.. code-block:: console

    Usage: main [options]

@@ -81,7 +81,7 @@ as of 2014. Alternately, the proxy can be stated manually or via a system such a
 Supervisord. Assuming that the ``$GALAXY_ROOT`` environment variable refers to the location of
 the Galaxy installation, the command for launching the proxy is:

-.. code-block::  console
+.. code-block:: console

    $ node $GALAXY_ROOT/lib/galaxy/web/proxy/js/lib/main.js --ip 0.0.0.0 \
        --port 8800 --sessions $GALAXY_ROOT/database/session_map.sqlite \
@@ -1,6 +1,6 @@
-=================================
+================================
 Containers for Tool Dependencies
-=================================
+================================

 Galaxy tools (also called wrappers) are able to use Conda packages
 (see more information in our `Galaxy Conda documentation`_) and Docker containers as dependency resolvers.
@@ -59,7 +59,7 @@ This will search for containers in the biocontainers organisation.


 Build all packages from bioconda from the last 24h
-^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

 The BioConda community is building a container for every package they create with a command similar to this.

@@ -70,7 +70,7 @@ The BioConda community is building a container for every package they create wit


 Building Docker containers for local Conda packages
-^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

 Conda packages can be tested with creating a busybox based container for this particular package in the following way.
 This also demonstrates how you can build a container locally and on-the-fly.
@@ -101,7 +101,7 @@ The ``--0`` indicates the build version of the conda package. It is recommended
 you will override already existing images. For Python Conda packages this extension might look like this ``--py35_1``.

 Build, test and push a conda-forge package to biocontainers
-^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

 > You need to have write access to the biocontainers repository

@@ -22,12 +22,10 @@ source_parsers = {
    '.md': CommonMarkParser,
 }

-####### REQUIRED GALAXY INCLUDES
+# REQUIRED GALAXY INCLUDES

 sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))

-#######
-
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
@@ -201,8 +199,8 @@ latex_elements = {
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-  ('index', 'Galaxy.tex', u'Galaxy Code Documentation',
-   u'Galaxy Team', 'manual'),
+    ('index', 'Galaxy.tex', u'Galaxy Code Documentation',
+     u'Galaxy Team', 'manual'),
 ]

 # The name of an image file (relative to this directory) to place at the top of
@@ -245,9 +243,9 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'Galaxy', u'Galaxy Code Documentation',
-   u'Galaxy Team', 'Galaxy', 'Data intensive biology for everyone.',
-   'Miscellaneous'),
+    ('index', 'Galaxy', u'Galaxy Code Documentation',
+     u'Galaxy Team', 'Galaxy', 'Data intensive biology for everyone.',
+     'Miscellaneous'),
 ]

 # Documents to append as an appendix to all manuals.
@@ -259,8 +257,8 @@ texinfo_documents = [
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
 #texinfo_show_urls = 'footnote'

-# -- ReadTheDocs.org Settings ------------------------------------------------

+# -- ReadTheDocs.org Settings ------------------------------------------------
 class Mock(object):
    def __init__(self, *args, **kwargs):
        pass
@@ -1,8 +1,8 @@
-BUILD RUNNER FOR GALAXY
-----------------------
+Build a job runner
+==================

-A walk through the steps of building a runner for Galaxy.
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+A walk through the steps of building a runner for Galaxy
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 In this tutorial, we would build the runner in a block by block fashion
 (like the building blocks), so we would divide the runner into
@@ -14,7 +14,7 @@ To learn more about the basics, please refer to:
 https://wiki.galaxyproject.org/Admin/GetGalaxy

 To explore existing runners, please refer to:
-https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/jobs/runner
+https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/jobs/runners

 What is required to make a runner for Galaxy?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -51,13 +51,12 @@ Implementation of parent class (galaxy.jobs.runners.\_\_init\_\_.py)
 -  .. rubric:: Class Inheritance structure
      :name: class-inheritance-structure

-   .. figure:: https://github.com/varunshankar/galaxy-godocker/raw/master/inherit.png
-      :alt: 
+   .. image:: inherit.png

 -  .. rubric:: The big picture!
      :name: the-big-picture-1

-   .. figure:: https://raw.githubusercontent.com/varunshankar/galaxy-godocker/master/runner_diag.png
+   .. image:: runner_diag.png

 The whole process is divided into different stages for understanding
 purpose.
@@ -218,7 +217,7 @@ Output params: AsynchronousJobState object

 Without going into much detail, assume there is a queue to track the status of every job. eg:

-.. image:: https://raw.githubusercontent.com/varunshankar/galaxy-godocker/master/queue.png
+.. image:: queue.png
    :align: center

 The galaxy framework updates the status of a job by iterating through the
@@ -229,7 +228,7 @@ copy the output files for the completed jobs.

 Updated result after an iteration (after invocation of check\_watched\_item 6 times):

-.. image:: https://raw.githubusercontent.com/varunshankar/galaxy-godocker/master/queue_b.png
+.. image:: queue_b.png
    :align: center


@@ -1,15 +1,18 @@
 Developer Documentation
 =======================

-There are two primary areas of documentation developers will be interested in:
+.. toctree::
+  :maxdepth: 1
+
+  schema
+  interactive_environments
+  build_a_job_runner
+  faq
+
+These are other primary areas of documentation developers will be interested in:

 - `Codebase Documentation`_ for developers of Galaxy
 - `API Documentation`_ for developers of third party tools interacting with Galaxy
- `Interactive Environments`_ for developers of Galaxy Interactive Environments
-
-Additionally there is an `FAQ`_ document for new Galaxy developers answering common "How Do I..." questions.

 .. _Codebase Documentation: ../lib/modules.html
 .. _API Documentation: ../api_doc.html
-.. _Interactive Environments: interactive_environments.html
-.. _FAQ: faq.html
@@ -52,13 +52,12 @@ Indices and tables
 * :ref:`search`

 Building this Documentation
-==========================
+===========================

 If you have your own copy of the Galaxy source code, you can also generate your own version of this documentation:

 ::

-    $ cd doc
-    $ make html
+    $ make -C doc/ html

 The generated documentation will be in ``doc/build/html/`` and can be viewed with a web browser.  Note that you will need to install Sphinx and a fair number of module dependencies before this will produce output.
@@ -225,18 +225,18 @@ galaxy.datatypes.converters.vcf_to_vcf_bgzip module
    :undoc-members:
    :show-inheritance:

-galaxy.datatypes.converters.bcf_bgzip_to_bcf module
---------------------------------------------------
+galaxy.datatypes.converters.bcf_bgzip_to_bcf_converter module
+-------------------------------------------------------------

-.. automodule:: galaxy.datatypes.converters.bcf_bgzip_to_bcf
+.. automodule:: galaxy.datatypes.converters.bcf_bgzip_to_bcf_converter
    :members:
    :undoc-members:
    :show-inheritance:

-galaxy.datatypes.converters.bcf_to_bcf_bgzip module
---------------------------------------------------
+galaxy.datatypes.converters.bcf_to_bcf_bgzip_converter module
+-------------------------------------------------------------

-.. automodule:: galaxy.datatypes.converters.bcf_to_bcf_bgzip
+.. automodule:: galaxy.datatypes.converters.bcf_to_bcf_bgzip_converter
    :members:
    :undoc-members:
    :show-inheritance:
@@ -14,7 +14,6 @@ Subpackages
    galaxy.jobs.actions
    galaxy.jobs.deferred
    galaxy.jobs.metrics
-    galaxy.jobs.rules
    galaxy.jobs.runners
    galaxy.jobs.splitters

@@ -22,7 +22,6 @@ Subpackages
    galaxy.forms
    galaxy.jobs
    galaxy.managers
-    galaxy.metadata
    galaxy.model
    galaxy.objectstore
    galaxy.openid
@@ -25,6 +25,14 @@ galaxy.tools.linters.command module
    :undoc-members:
    :show-inheritance:

+galaxy.tools.linters.general module
+-----------------------------------
+
+.. automodule:: galaxy.tools.linters.general
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 galaxy.tools.linters.help module
 --------------------------------

@@ -49,6 +57,14 @@ galaxy.tools.linters.outputs module
    :undoc-members:
    :show-inheritance:

+galaxy.tools.linters.stdio module
+---------------------------------
+
+.. automodule:: galaxy.tools.linters.stdio
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 galaxy.tools.linters.tests module
 ---------------------------------

@@ -57,12 +73,10 @@ galaxy.tools.linters.tests module
    :undoc-members:
    :show-inheritance:

-galaxy.tools.linters.top_level module
+galaxy.tools.linters.xml_order module
 -------------------------------------

-.. automodule:: galaxy.tools.linters.top_level
+.. automodule:: galaxy.tools.linters.xml_order
    :members:
    :undoc-members:
    :show-inheritance:
-
-
@@ -65,14 +65,6 @@ galaxy.tools.parameters.meta module
    :undoc-members:
    :show-inheritance:

-galaxy.tools.parameters.output module
-------------------------------------
-
-.. automodule:: galaxy.tools.parameters.output
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 galaxy.tools.parameters.output_collect module
 ---------------------------------------------

@@ -105,4 +97,10 @@ galaxy.tools.parameters.wrapped module
    :undoc-members:
    :show-inheritance:

+galaxy.tools.parameters.wrapped_json module
+-------------------------------------------

+.. automodule:: galaxy.tools.parameters.wrapped_json
+    :members:
+    :undoc-members:
+    :show-inheritance:
@@ -1,8 +0,0 @@
-galaxy.util.backports.importlib package
-=======================================
-
-.. automodule:: galaxy.util.backports.importlib
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
@@ -5,11 +5,3 @@ galaxy.util.backports package
    :members:
    :undoc-members:
    :show-inheritance:
-
-Subpackages
-----------
-
-.. toctree::
-
-    galaxy.util.backports.importlib
-
@@ -42,6 +42,14 @@ galaxy.util.bunch module
    :undoc-members:
    :show-inheritance:

+galaxy.util.checkers module
+---------------------------
+
+.. automodule:: galaxy.util.checkers
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 galaxy.util.dbkeys module
 -------------------------

@@ -50,26 +58,10 @@ galaxy.util.dbkeys module
    :undoc-members:
    :show-inheritance:

-galaxy.util.debugging module
----------------------------
+galaxy.util.dictifiable module
+------------------------------

-.. automodule:: galaxy.util.debugging
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.util.dictobj module
--------------------------
-
-.. automodule:: galaxy.util.dictobj
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-galaxy.util.directory_hash module
---------------------------------
-
-.. automodule:: galaxy.util.directory_hash
+.. automodule:: galaxy.util.dictifiable
    :members:
    :undoc-members:
    :show-inheritance:
@@ -82,6 +74,14 @@ galaxy.util.expressions module
    :undoc-members:
    :show-inheritance:

+galaxy.util.filelock module
+---------------------------
+
+.. automodule:: galaxy.util.filelock
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 galaxy.util.hash_util module
 ----------------------------

@@ -98,6 +98,14 @@ galaxy.util.heartbeat module
    :undoc-members:
    :show-inheritance:

+galaxy.util.image_util module
+-----------------------------
+
+.. automodule:: galaxy.util.image_util
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 galaxy.util.inflection module
 -----------------------------

@@ -130,10 +138,10 @@ galaxy.util.lazy_process module
    :undoc-members:
    :show-inheritance:

-galaxy.util.lrucache module
---------------------------
+galaxy.util.multi_byte module
+-----------------------------

-.. automodule:: galaxy.util.lrucache
+.. automodule:: galaxy.util.multi_byte
    :members:
    :undoc-members:
    :show-inheritance:
@@ -178,6 +186,14 @@ galaxy.util.plugin_config module
    :undoc-members:
    :show-inheritance:

+galaxy.util.postfork module
+---------------------------
+
+.. automodule:: galaxy.util.postfork
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 galaxy.util.properties module
 -----------------------------

@@ -266,6 +282,14 @@ galaxy.util.topsort module
    :undoc-members:
    :show-inheritance:

+galaxy.util.ucsc module
+-----------------------
+
+.. automodule:: galaxy.util.ucsc
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 galaxy.util.validation module
 -----------------------------

@@ -1,9 +1,8 @@

 .. to_doc

-------------------------------
 16.04
-------------------------------
+===============================

 .. announce_start

@@ -65,6 +65,7 @@ See `our wiki <https://wiki.galaxyproject.org/Develop/SourceCode>`__ for additio

 Security
 ===========================================================
+
 TL;DR
 **Only Tool Sheds newer than 16.01 should be deployed from now on.**
 (with commit 449098d8b14b45269be106f6410c0b9145c51d50 from Mar 30 present)
@@ -81,7 +82,7 @@ Deprecation Notices
 ===========================================================

 API deprecations
-~~~~~~~~~~~~~~~~
+----------------

 API for history contents, index:
 * **types**: is no longer a valid parameter but accessible using ``?q=history_content_type&qv=[dataset | dataset_collection]``
@@ -96,7 +97,7 @@ API histories (removed from the available serialized data on all calls):
  * **state, state_details, state_ids** - can be replaced by specifically requesting a single array of contents, each containing :code:`{ id, state, deleted, visible }`

 Galaxy no longer on Bitbucket
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+-----------------------------

 Galaxy moved its code and development activities from Bitbucket to GitHub in early 2015. Since this time, releases have been mirrored back to Bitbucket. However, after this release, no new changes will be pushed to Bitbucket. Anyone still receiving updates to a Galaxy server via Bitbucket and Mercurial should switch to GitHub and Git. This can be done using the following process:

@@ -1,9 +1,8 @@

 .. to_doc

-------------------------------
 16.07
-------------------------------
+===============================

 .. announce_start

@@ -111,7 +110,7 @@ Enhancements
 * Fix running tool tests if remote user middleware is enabled.
  `Pull Request 2173`_
 * Improve handling of a missing R environment during Tool Shed dependency
-   installations.
+  installations.
  `Pull Request 2215`_
 * Quote some parameters in the trim tool command
  (thanks to `@nsoranzo <https://github.com/nsoranzo>`__.)
@@ -4,6 +4,7 @@ Releases
 .. toctree::
   :maxdepth: 1

+   16.10_announce
   16.07_announce
   16.04_announce
   16.01_announce
@@ -1170,10 +1170,13 @@ class OxliCountGraph(OxliBinary):
    """
    OxliCountGraph starts with "OXLI" + one byte version number +
    8-bit binary '1'
-    Test file generated via `load-into-counting.py --n_tables 1 \
-            --max-tablesize 1 oxli_countgraph.oxlicg \
-            khmer/tests/test-data/100-reads.fq.bz2`
+    Test file generated via::
+
+        load-into-counting.py --n_tables 1 --max-tablesize 1 \\
+            oxli_countgraph.oxlicg khmer/tests/test-data/100-reads.fq.bz2
+
    using khmer 2.0
+
    >>> from galaxy.datatypes.sniff import get_test_fname
    >>> fname = get_test_fname( 'sequence.csfasta' )
    >>> OxliCountGraph().sniff( fname )
@@ -1194,10 +1197,13 @@ class OxliNodeGraph(OxliBinary):
    """
    OxliNodeGraph starts with "OXLI" + one byte version number +
    8-bit binary '2'
-    Test file generated via `load-graph.py --n_tables 1 \
-            --max-tablesize 1 oxli_nodegraph.oxling \
-            khmer/tests/test-data/100-reads.fq.bz2`
+    Test file generated via::
+
+        load-graph.py --n_tables 1 --max-tablesize 1 oxli_nodegraph.oxling \\
+            khmer/tests/test-data/100-reads.fq.bz2
+
    using khmer 2.0
+
    >>> from galaxy.datatypes.sniff import get_test_fname
    >>> fname = get_test_fname( 'sequence.csfasta' )
    >>> OxliNodeGraph().sniff( fname )
@@ -1218,11 +1224,14 @@ class OxliTagSet(OxliBinary):
    """
    OxliTagSet starts with "OXLI" + one byte version number +
    8-bit binary '3'
-    Test file generated via `load-graph.py --n_tables 1 \
-            --max-tablesize 1 oxli_nodegraph.oxling \
-            khmer/tests/test-data/100-reads.fq.bz2; \
-            mv oxli_nodegraph.oxling.tagset oxli_tagset.oxlits`
+    Test file generated via::
+
+        load-graph.py --n_tables 1 --max-tablesize 1 oxli_nodegraph.oxling \\
+            khmer/tests/test-data/100-reads.fq.bz2;
+        mv oxli_nodegraph.oxling.tagset oxli_tagset.oxlits
+
    using khmer 2.0
+
    >>> from galaxy.datatypes.sniff import get_test_fname
    >>> fname = get_test_fname( 'sequence.csfasta' )
    >>> OxliTagSet().sniff( fname )
@@ -1244,6 +1253,7 @@ class OxliStopTags(OxliBinary):
    8-bit binary '4'
    Test file adapted from khmer 2.0's
    "khmer/tests/test-data/goodversion-k32.stoptags"
+
    >>> from galaxy.datatypes.sniff import get_test_fname
    >>> fname = get_test_fname( 'sequence.csfasta' )
    >>> OxliStopTags().sniff( fname )
@@ -1264,11 +1274,14 @@ class OxliSubset(OxliBinary):
    """
    OxliSubset starts with "OXLI" + one byte version number +
    8-bit binary '5'
-    Test file generated via `load-graph.py -k 20 example \
-            tests/test-data/random-20-a.fa; \
-            partition-graph.py example; \
-            mv example.subset.0.pmap oxli_subset.oxliss`
+    Test file generated via::
+
+        load-graph.py -k 20 example tests/test-data/random-20-a.fa;
+        partition-graph.py example;
+        mv example.subset.0.pmap oxli_subset.oxliss
+
    using khmer 2.0
+
    >>> from galaxy.datatypes.sniff import get_test_fname
    >>> fname = get_test_fname( 'sequence.csfasta' )
    >>> OxliSubset().sniff( fname )
@@ -1288,11 +1301,15 @@ class OxliGraphLabels(OxliBinary):
    """
    OxliGraphLabels starts with "OXLI" + one byte version number +
    8-bit binary '6'
-    Test file generated via `python -c "from khmer import GraphLabels; \
-        gl = GraphLabels(20, 1e7, 4); gl.consume_fasta_and_tag_with_labels(
-            'tests/test-data/test-labels.fa'); \
-        gl.save_labels_and_tags('oxli_graphlabels.oxligl')"`
+    Test file generated via::
+
+        python -c "from khmer import GraphLabels; \\
+            gl = GraphLabels(20, 1e7, 4); \\
+            gl.consume_fasta_and_tag_with_labels('tests/test-data/test-labels.fa'); \\
+            gl.save_labels_and_tags('oxli_graphlabels.oxligl')"
+
    using khmer 2.0
+
    >>> from galaxy.datatypes.sniff import get_test_fname
    >>> fname = get_test_fname( 'sequence.csfasta' )
    >>> OxliGraphLabels().sniff( fname )
@@ -1,4 +1,4 @@
-"""Module proxies :module:`galaxy.util.checkers` for backward compatibility.
+"""Module proxies :mod:`galaxy.util.checkers` for backward compatibility.

 External datatypes may make use of these functions.
 """
@@ -134,7 +134,7 @@ Binary.register_sniffable_binary_format("plybinary", "plybinary", PlyBinary)


 class Vtk(object):
-    """
+    r"""
    The Visualization Toolkit provides a number of source and writer objects to
    read and write popular data file formats. The Visualization Toolkit also
    provides some of its own file formats.
@@ -151,8 +151,8 @@ class Vtk(object):
    i.e., the numbers that define points coordinates, scalars, cell indices, and
    so forth.

-    Binary data must be placed into the file immediately after the newline (\n)
-    character from the previous ASCII keyword and parameter sequence.
+    Binary data must be placed into the file immediately after the newline
+    ('\\n') character from the previous ASCII keyword and parameter sequence.

    TODO: only legacy formats are currently supported and support for XML formats
    should be added.
@@ -66,14 +66,13 @@ class Data( object ):
    'test'
    >>> type( DataTest.metadata_spec.test.param )
    <class 'galaxy.model.metadata.MetadataParameter'>
-
    """
    edam_data = "data_0006"
    edam_format = "format_1915"
    # Data is not chunkable by default.
    CHUNKABLE = False

-    #: dictionary of metadata fields for this datatype::
+    #: Dictionary of metadata fields for this datatype
    metadata_spec = None

    # Add metadata elements
@@ -855,7 +854,7 @@ class Text( Data ):
    @dataproviders.decorators.dataprovider_factory( 'line', dataproviders.line.FilteredLineDataProvider.settings )
    def line_dataprovider( self, dataset, **settings ):
        """
-        Returns an iterator over the dataset's lines (that have been `strip`ed)
+        Returns an iterator over the dataset's lines (that have been stripped)
        optionally excluding blank lines and lines that start with a comment character.
        """
        dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
@@ -962,10 +961,9 @@ def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skip
    """
    Returns the first LINE_COUNT lines wrapped to WIDTH

-    ## >>> fname = get_test_fname('4.bed')
-    ## >>> get_file_peek(fname)
-    ## 'chr22    30128507    31828507    uc003bnx.1_cds_2_0_chr22_29227_f    0    +\n'
-
+    >>> fname = get_test_fname('4.bed')
+    >>> get_file_peek(fname, LINE_COUNT=1)
+    u'chr22\\t30128507\\t31828507\\tuc003bnx.1_cds_2_0_chr22_29227_f\\t0\\t+\\n'
    """
    # Set size for file.readline() to a negative number to force it to
    # read until either a newline or EOF.  Needed for datasets with very
@@ -6,7 +6,6 @@ consumer datum by datum.

 As well as subclassing and overriding to get the proper data, Dataproviders
 can be piped from one to the other.
-..example::

 .. note:: be careful to NOT pipe providers into subclasses of those providers.
    Subclasses provide all the functionality of their superclasses,
@@ -78,8 +78,8 @@ class ColumnarDataProvider( line.RegexLineDataProvider ):
            Optional: defaults to the tab character.
        :type deliminator: str

-        .. note: that the subclass constructors are passed kwargs - so they're
-        params (limit, offset, etc.) are also applicable here.
+        .. note:: that the subclass constructors are passed kwargs - so they're
+            params (limit, offset, etc.) are also applicable here.
        """
        # TODO: other columnar formats: csv, etc.
        super( ColumnarDataProvider, self ).__init__( source, **kwargs )
@@ -141,9 +141,13 @@ class ColumnarDataProvider( line.RegexLineDataProvider ):

        The function will compare the column at index `column` against `val`
        using the given op where op is one of:
-            lt: less than, le: less than or equal to,
-            eq: equal to, ne: not equal to,
-            ge: greather than or equal to, gt: greater than
+
+        - lt: less than
+        - le: less than or equal to
+        - eq: equal to
+        - ne: not equal to
+        - ge: greather than or equal to
+        - gt: greater than

        `val` is cast as float here and will return None if there's a parsing error.
        """
@@ -173,9 +177,10 @@ class ColumnarDataProvider( line.RegexLineDataProvider ):

        The function will compare the column at index `column` against `val`
        using the given op where op is one of:
-            eq: exactly matches,
-            has: the column contains the substring `val`,
-            re: the column matches the regular expression in `val`
+
+        - eq: exactly matches
+        - has: the column contains the substring `val`
+        - re: the column matches the regular expression in `val`
        """
        if 'eq' == op:
            return lambda d: d[column] == val
@@ -195,8 +200,9 @@ class ColumnarDataProvider( line.RegexLineDataProvider ):

        The function will compare the column at index `column` against `val`
        using the given op where op is one of:
-            eq: the list `val` exactly matches the list in the column,
-            has: the list in the column contains the sublist `val`,
+
+        - eq: the list `val` exactly matches the list in the column
+        - has: the list in the column contains the sublist `val`
        """
        if 'eq' == op:
            val = self.parse_value( val, 'list' )
@@ -210,8 +216,9 @@ class ColumnarDataProvider( line.RegexLineDataProvider ):
        Return parser dictionary keyed for each columnar type
        (as defined in datatypes).

-        .. note: primitives only by default (str, int, float, boolean, None).
+        .. note:: primitives only by default (str, int, float, boolean, None).
            Other (more complex) types are retrieved as strings.
+
        :returns: a dictionary of the form:
            `{ <parser type name> : <function used to parse type> }`
        """
@@ -324,8 +331,8 @@ class DictDataProvider( ColumnarDataProvider ):
    A combination use of both `column_names` and `indeces` allows 'picking'
    key/value pairs from the source.

-    .. note: that the subclass constructors are passed kwargs - so they're
-    params (limit, offset, etc.) are also applicable here.
+    .. note:: The subclass constructors are passed kwargs - so their
+        params (limit, offset, etc.) are also applicable here.
    """
    settings = {
        'column_names'  : 'list:str',
@@ -56,6 +56,7 @@ class DatasetDataProvider( base.DataProvider ):
    def get_column_metadata_from_dataset( cls, dataset ):
        """
        Convenience class method to get column metadata from a dataset.
+
        :returns: a dictionary of `column_count`, `column_types`, and `column_names`
            if they're available, setting each to `None` if not.
        """
@@ -69,6 +70,7 @@ class DatasetDataProvider( base.DataProvider ):
    def get_metadata_column_types( self, indeces=None ):
        """
        Return the list of `column_types` for this dataset or `None` if unavailable.
+
        :param indeces: the indeces for the columns of which to return the types.
            Optional: defaults to None (return all types)
        :type indeces: list of ints
@@ -88,6 +90,7 @@ class DatasetDataProvider( base.DataProvider ):
    def get_metadata_column_names( self, indeces=None ):
        """
        Return the list of `column_names` for this dataset or `None` if unavailable.
+
        :param indeces: the indeces for the columns of which to return the names.
            Optional: defaults to None (return all names)
        :type indeces: list of ints
@@ -108,8 +111,10 @@ class DatasetDataProvider( base.DataProvider ):
    def get_indeces_by_column_names( self, list_of_column_names ):
        """
        Return the list of column indeces when given a list of column_names.
+
        :param list_of_column_names: the names of the columns of which to get indeces.
        :type list_of_column_names: list of strs
+
        :raises KeyError: if column_names are not found
        :raises ValueError: if an entry in list_of_column_names is not in column_names
        """
@@ -399,7 +404,8 @@ class IntervalDataProvider( column.ColumnarDataProvider ):
 #   WITHOUT reading the entire seq into memory - possibly apply some version of limit/offset
 class FastaDataProvider( base.FilteredDataProvider ):
    """
-    Class that returns fasta format data in a list of maps of the form:
+    Class that returns fasta format data in a list of maps of the form::
+
        {
            id: <fasta header id>,
            sequence: <joined lines of nucleotide/amino data>
@@ -432,7 +438,8 @@ class FastaDataProvider( base.FilteredDataProvider ):

 class TwoBitFastaDataProvider( DatasetDataProvider ):
    """
-    Class that returns fasta format data in a list of maps of the form:
+    Class that returns fasta format data in a list of maps of the form::
+
        {
            id: <fasta header id>,
            sequence: <joined lines of nucleotide/amino data>
@@ -32,25 +32,24 @@ def has_dataproviders( cls ):
    in the class.

    This allows a class to maintain a name -> method map, effectively
-    'registering' dataprovider factory methods.
+    'registering' dataprovider factory methods::

-    .. example::
-    @has_dataproviders
-    class MyDtype( data.Data ):
+        @has_dataproviders
+        class MyDtype( data.Data ):

-        @dataprovider_factory( 'bler' )
-        def provide_some_bler( self, dataset, **settings ):
-            '''blerblerbler'''
-            dataset_source = providers.DatasetDataProvider( dataset )
-            # ... chain other, intermidiate providers here
-            return providers.BlerDataProvider( dataset_source, **settings )
+            @dataprovider_factory( 'bler' )
+            def provide_some_bler( self, dataset, **settings ):
+                '''blerblerbler'''
+                dataset_source = providers.DatasetDataProvider( dataset )
+                # ... chain other, intermidiate providers here
+                return providers.BlerDataProvider( dataset_source, **settings )

-    # use the base method in data.Data
-    provider = dataset.datatype.dataprovider( dataset, 'bler',
-                                              my_setting='blah', ... )
-    # OR directly from the map
-    provider = dataset.datatype.dataproviders[ 'bler' ]( dataset,
-                                                         my_setting='blah', ... )
+        # use the base method in data.Data
+        provider = dataset.datatype.dataprovider( dataset, 'bler',
+                                                  my_setting='blah', ... )
+        # OR directly from the map
+        provider = dataset.datatype.dataproviders[ 'bler' ]( dataset,
+                                                             my_setting='blah', ... )
    """
    # init the class dataproviders map if necc.
    if not hasattr( cls, _DATAPROVIDER_CLASS_MAP_KEY ):
@@ -82,15 +81,15 @@ def dataprovider_factory( name, settings=None ):
    function to parse query strings to __init__ arguments as the
    `parse_query_string_settings` attribute of the factory function.

-    An example use of the `parse_query_string_settings`:
-    ..example::
-    kwargs = dataset.datatype.dataproviders[ provider ].parse_query_string_settings( query_kwargs )
-    return list( dataset.datatype.dataprovider( dataset, provider, **kwargs ) )
+    An example use of the `parse_query_string_settings`::
+
+        kwargs = dataset.datatype.dataproviders[ provider ].parse_query_string_settings( query_kwargs )
+        return list( dataset.datatype.dataprovider( dataset, provider, **kwargs ) )

    :param name: what name/key to register the factory under in `cls.dataproviders`
    :type name: any hashable var
    :param settings: dictionary containing key/type pairs for parsing query strings
-    to __init__ arguments
+        to __init__ arguments
    :type settings: dictionary
    """
    # TODO:?? use *args for settings allowing mulitple dictionaries
@@ -88,7 +88,7 @@ class RegexLineDataProvider( FilteredLineDataProvider ):
    of regexs.

    .. note:: the regex matches are effectively OR'd (if **any** regex matches
-    the line it is considered valid and will be provided).
+        the line it is considered valid and will be provided).
    """
    settings = {
        'regex_list'    : 'list:escaped',
@@ -1007,19 +1007,28 @@ class DotBracket ( Sequence ):
        Galaxy Dbn (Dot-Bracket notation) rules:

        * The first non-empty line is a header line: no comment lines are allowed.
+
          * A header line starts with a '>' symbol and continues with 0 or multiple symbols until the line ends.
+
        * The second non-empty line is a sequence line.
-          * A sequence line may only include chars that match the Fasta format (https://en.wikipedia.org/wiki/FASTA_format#Sequence_representation) symbols for nucleotides: ACGTURYKMSWBDHVN, and may thus not include whitespaces.
+
+          * A sequence line may only include chars that match the FASTA format (https://en.wikipedia.org/wiki/FASTA_format#Sequence_representation) symbols for nucleotides: ACGTURYKMSWBDHVN, and may thus not include whitespaces.
          * A sequence line has no prefix and no suffix.
          * A sequence line is case insensitive.
+
        * The third non-empty line is a structure (Dot-Bracket) line and only describes the 2D structure of the sequence above it.
+
          * A structure line must consist of the following chars: '.{}[]()'.
          * A structure line must be of the same length as the sequence line, and each char represents the structure of the nucleotide above it.
          * A structure line has no prefix and no suffix.
          * A nucleotide pairs with only 1 or 0 other nucleotides.
+
            * In a structure line, the number of '(' symbols equals the number of ')' symbols, the number of '[' symbols equals the number of ']' symbols and the number of '{' symbols equals the number of '}' symbols.
+
        * The format accepts multiple entries per file, given that each entry is provided as three lines: the header, sequence and structure line.
+
            * Sniffing is only applied on the first entry.
+
        * Empty lines are allowed.
         """

@@ -1058,31 +1058,31 @@ class ConnectivityTable( Tabular ):
        The ConnectivityTable (CT) is a file format used for describing
        RNA 2D structures by tools including MFOLD, UNAFOLD and
        the RNAStructure package. The tabular file format is defined as
-        follows:
+        follows::

-5	energy = -12.3	sequence name
-1	G	0	2	0	1
-2	A	1	3	0	2
-3	A	2	4	0	3
-4	A	3	5	0	4
-5	C	4	6	1	5
+            5	energy = -12.3	sequence name
+            1	G	0	2	0	1
+            2	A	1	3	0	2
+            3	A	2	4	0	3
+            4	A	3	5	0	4
+            5	C	4	6	1	5

        The links given at the edam ontology page do not indicate what
        type of separator is used (space or tab) while different
        implementations exist. The implementation that uses spaces as
-        separator (implemented in RNAStructure) is as follows:
+        separator (implemented in RNAStructure) is as follows::

-   10    ENERGY = -34.8  seqname
-    1 G       0    2    9    1
-    2 G       1    3    8    2
-    3 G       2    4    7    3
-    4 a       3    5    0    4
-    5 a       4    6    0    5
-    6 a       5    7    0    6
-    7 C       6    8    3    7
-    8 C       7    9    2    8
-    9 C       8   10    1    9
-   10 a       9    0    0   10
+            10    ENERGY = -34.8  seqname
+            1 G       0    2    9    1
+            2 G       1    3    8    2
+            3 G       2    4    7    3
+            4 a       3    5    0    4
+            5 a       4    6    0    5
+            6 a       5    7    0    6
+            7 C       6    8    3    7
+            8 C       7    9    2    8
+            9 C       8   10    1    9
+            10 a       9    0    0   10
        """

        i = 0
@@ -541,9 +541,8 @@ class JobConfiguration( object ):
        a list of IDs, the JobToolConfigurations for the first id in ``ids``
        matching a tool definition.

-        .. note::
-
-            You should not mix tool shed tool IDs, versionless tool shed IDs, and tool config tool IDs that refer to the same tool.
+        .. note:: You should not mix tool shed tool IDs, versionless tool shed
+             IDs, and tool config tool IDs that refer to the same tool.

        :param ids: Tool ID or IDs to fetch the JobToolConfiguration of.
        :type ids: list or str.
@@ -1,4 +1,4 @@
-"""This module describes the abstract interface for :class:`InstrumentPlugin`s.
+"""This module describes the abstract interface for :class:`InstrumentPlugin`.

 These are responsible for collecting and formatting a coherent set of metrics.
 """
@@ -6,26 +6,26 @@ Encapsulates the intersection of trans (or trans.sa_session), models,
 and Controllers.

 Responsibilities:
-    model operations that involve the trans/sa_session (CRUD)
-    security:
-        ownership, accessibility
-    common aspect-oriented operations via new mixins:
-        sharable, annotatable, tagable, ratable
+
+- model operations that involve the trans/sa_session (CRUD)
+- security: ownership, accessibility
+- common aspect-oriented operations via new mixins: sharable, annotatable,
+  tagable, ratable

 Not responsible for:
-    encoding/decoding ids
-    any http gobblygook
-    formatting of returned data (always python structures)
-    formatting of raised errors
+
+- encoding/decoding ids
+- any http gobblygook
+- formatting of returned data (always python structures)
+- formatting of raised errors

 The goal is to have Controllers only handle:
-    query-string/payload parsing and encoding/decoding ids
-    http
-    return formatting

-and:
-    control, improve namespacing in Controllers
-    DRY for Controller ops (define here - use in both UI/API Controllers)
+- query-string/payload parsing and encoding/decoding ids
+- http
+- return formatting
+- control, improve namespacing in Controllers
+- DRY for Controller ops (define here - use in both UI/API Controllers)

 In other words, 'Business logic' independent of web transactions/user context
 (trans) should be pushed into models - but logic that requires the context
@@ -4,9 +4,10 @@ defines a base ModelManager, ModelSerializer, and ModelDeserializer.

 ModelManagers are used for operations on models that occur outside the scope of
 a single model object, such as:
-    - object creation
-    - object lookup
-    - interactions between 2+ objects of different model classes
+
+- object creation
+- object lookup
+- interactions between 2+ objects of different model classes

 (Since these were to replace model Mixins from
 web/framework/base/controller.py the rule of thumb used there also generally
@@ -903,10 +904,12 @@ class ModelFilterParser( HasAModelManager ):
    """
    Converts string tuples (partially converted query string params) of
    attr, op, val into either:
-        - ORM based filters (filters that can be applied by the ORM at the SQL
-        level) or
-        - functional filters (filters that use derived values or values not
-        within the SQL tables)
+
+    - ORM based filters (filters that can be applied by the ORM at the SQL
+      level) or
+    - functional filters (filters that use derived values or values not
+      within the SQL tables)
+
    These filters can then be applied to queries.

    This abstraction allows 'smarter' application of limit and offset at either the
@@ -236,7 +236,8 @@ class DatasetDeserializer( base.ModelDeserializer, deletable.PurgableDeserialize
    def deserialize_permissions( self, dataset, key, permissions, user=None, **context ):
        """
        Create permissions for each list of encoded role ids in the (validated)
-        `permissions` dictionary, where `permissions` is in the form:
+        `permissions` dictionary, where `permissions` is in the form::
+
            { 'manage': [ <role id 1>, ... ], 'access': [ <role id 2>, ... ] }
        """
        self.manager.permissions.manage.error_unless_permitted( dataset, user )
@@ -355,7 +356,7 @@ class DatasetAssociationManager( base.ModelManager,
        """
        Return True if this hda/ldda is a composite type dataset.

-        .. note: see also (whereever we keep information on composite datatypes?)
+        .. note:: see also (whereever we keep information on composite datatypes?)
        """
        return dataset_assoc.extension in self.app.datatypes_registry.get_composite_extensions()

@@ -33,7 +33,7 @@ class LibraryManager( object ):
        :type   check_accessible:         bool

        :returns:   the requested library
-        :rtype:     Library
+        :rtype:     galaxy.model.Library
        """
        try:
            library = trans.sa_session.query( trans.app.model.Library ).filter( trans.app.model.Library.table.c.id == decoded_library_id ).one()
@@ -145,8 +145,8 @@ class LibraryManager( object ):
        """
        Check if library is accessible to user.

-        :param  folder:                  library
-        :type   folder:                  Library
+        :param  library:                 library
+        :type   library:                 galaxy.model.Library
        :param  check_accessible:        flag whether to check that user can access library
        :type   check_accessible:        bool

@@ -176,7 +176,7 @@ class LibraryManager( object ):
        Return library data in the form of a dictionary.

        :param  library:       library
-        :type   library:       Library
+        :type   library:       galaxy.model.Library

        :returns:   dict with data about the library
        :rtype:     dictionary
@@ -201,7 +201,7 @@ class LibraryManager( object ):
        Load all permissions currently related to the given library.

        :param  library:      the model object
-        :type   library:      Library
+        :type   library:      galaxy.model.Library

        :rtype:     dictionary
        :returns:   dict of current roles for all available permission types
@@ -13,7 +13,7 @@ class RBACPermissionFailedException( galaxy.exceptions.InsufficientPermissionsEx

 class RBACPermission( object ):
    """
-    Base class for wrangling/controlling the permissions ORM models (*Permissions, Roles)
+    Base class for wrangling/controlling the permissions ORM models (\*Permissions, Roles)
    that control which users can perform certain actions on their associated models
    (Libraries, Datasets).
    """
@@ -68,8 +68,9 @@ class DatasetRBACPermission( RBACPermission ):

    DatasetPermissions are typed (but not polymorphic themselves) by a string 'action'.
    There are two types:
-        - manage permissions : can a role manage the permissions on a dataset
-        - access : can a role read/look at/copy a dataset
+
+    - manage permissions : can a role manage the permissions on a dataset
+    - access : can a role read/look at/copy a dataset
    """
    permissions_class = model.DatasetPermissions
    action_name = None
@@ -33,7 +33,7 @@ class RoleManager( base.ModelManager ):
        :type   decoded_role_id:      int

        :returns:   the loaded Role object
-        :rtype:     Role
+        :rtype:     galaxy.model.Role

        :raises: InconsistentDatabase, RequestParameterInvalidException, InternalServerError
        """
@@ -268,6 +268,7 @@ class TextToolParameter( ToolParameter ):
 class IntegerToolParameter( TextToolParameter ):
    """
    Parameter that takes an integer value.
+
    >>> from galaxy.util.bunch import Bunch
    >>> trans = Bunch( history=Bunch(), workflow_building_mode=True )
    >>> p = IntegerToolParameter( None, XML( '<param name="_name" type="integer" value="10" />' ) )
@@ -343,6 +344,7 @@ class IntegerToolParameter( TextToolParameter ):
 class FloatToolParameter( TextToolParameter ):
    """
    Parameter that takes a real number value.
+
    >>> from galaxy.util.bunch import Bunch
    >>> trans = Bunch( history=Bunch(), workflow_building_mode=True )
    >>> p = FloatToolParameter( None, XML( '<param name="_name" type="float" value="3.141592" />' ) )
@@ -74,7 +74,7 @@ the tool menu immediately following the hyperlink for the tool (based on the
        <xs:element name="stdio" type="Stdio" minOccurs="0"/>
        <xs:element name="help" type="xs:string" minOccurs="0">
          <xs:annotation gxdocs:best_practices="help-tag">
-            <xs:documentation xml:lang="en"><![CDATA[This tag set includes all of the necessary details of how to use the tool. This tag set should be included as the next to the last tag set, before citations, in the tool config. Tool help is written in reStructuredText. Included here is only an overview of a subset of features. For more information see http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html.
+            <xs:documentation xml:lang="en"><![CDATA[This tag set includes all of the necessary details of how to use the tool. This tag set should be included as the next to the last tag set, before citations, in the tool config. Tool help is written in reStructuredText. Included here is only an overview of a subset of features. For more information see http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html .

 tag | details
 --- | -------
@@ -504,13 +504,13 @@ def get_field_components_options( dataset, field_name ):

 Container tag set to specify tests via the <test> tag sets. Any number of tests can be included,
 and each test is wrapped within separate <test> tag sets. Functional tests are
-executed via [Planemo](http://planemo.readthedocs.io/) or the
+executed via [Planemo](https://planemo.readthedocs.io/) or the
 [run_tests.sh](https://github.com/galaxyproject/galaxy/blob/dev/run_tests.sh)
 shell script distributed with Galaxy.

 The documentation contained here is mostly reference documentation, for
 tutorials on writing tool tests please check out Planemo's
-[Test-Driven Development](http://planemo.readthedocs.io/en/latest/writing_advanced.html#test-driven-development)
+[Test-Driven Development](https://planemo.readthedocs.io/en/latest/writing_advanced.html#test-driven-development)
 documentation or the much older wiki content for
 [WritingTests](https://wiki.galaxyproject.org/Admin/Tools/WritingTests).

@@ -928,7 +928,7 @@ associated input ``repeat``.</xs:documentation>
      <xs:documentation xml:lang="en"><![CDATA[

 This tag set defines the tool's input parameters for executing the tool via the
-functional test framework. See [``test``'s](#tool|tests|test) documentation for
+functional test framework. See [test](#tool-tests-test) documentation for
 some simple examples of parameters.

 ]]></xs:documentation>
@@ -1216,7 +1216,7 @@ This directive specifies a test for an output's discovered dataset. It acts as a
 ### Example

 The functional test tool
-multi_output_assign_primary.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/multi_output_assign_primary.xml)
+[multi_output_assign_primary.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/multi_output_assign_primary.xml)
 provides a demonstration of using this tag.

 ```xml
@@ -1299,7 +1299,7 @@ tool demonstrates basic usage of an ``output_collection`` test expectation.

 The [CWPair2](https://github.com/galaxyproject/tools-iuc/blob/master/tools/cwpair2/cwpair2.xml)
 tool demonstrates that ``element``s can specify a ``compare`` attribute just
-like [``output``](#tool|tests|test|output).
+like [output](#tool-tests-test-output).

 ```xml
 <test>
@@ -1326,7 +1326,7 @@ test tool demonstrates the use of nested ``element`` directives as described
 above. Notice also that it tests the output with ``assert_contents`` instead of
 supplying a ``file`` attribute. Like hinted at with with ``compare`` attribute
 above, the ``element`` tag can specify any of the test attributes that apply to
-the [``output``](#tool|tests|test|output) (e.g. ``md5``, ``compare``, ``diff``,
+the [output](#tool-tests-test-output) (e.g. ``md5``, ``compare``, ``diff``,
 etc...).

 ```xml
@@ -1412,7 +1412,7 @@ many of the default assertion tags that come with Galaxy and examples of each
 can be found below.

 The implementation of these tags are simply Python functions defined in the
-[``galaxy.tools.verify.asserts``](https://github.com/galaxyproject/galaxy/tree/dev/lib/galaxy/tools/verify/asserts)
+[galaxy.tools.verify.asserts](https://github.com/galaxyproject/galaxy/tree/dev/lib/galaxy/tools/verify/asserts)
 module.
 ]]>
      </xs:documentation>
@@ -1511,7 +1511,7 @@ module.
    <xs:annotation>
      <xs:documentation xml:lang="en"><![CDATA[Consists of all tag sets that define the
 tool's input parameters. Each ``<param>`` tag within the ``<inputs>`` tag set
-maps to a command line parameter within the [``command``](#tool|command) tag. Most
+maps to a command line parameter within the [command](#tool-command) tag. Most
 tools will not need to specify any attributes on this tag itself.]]>
 </xs:documentation>
    </xs:annotation>
@@ -1602,7 +1602,7 @@ statement. A good example tool that demonstrates many conditional parameters is
 </conditional>
 ```

-The first directive following the conditional is a [``param``](#tool|inputs|param),
+The first directive following the conditional is a [param](#tool-inputs-param),
 this param must be of type ``select`` or ``boolean``. Depending on the value a
 user selects for this "test" parameter - different UI elements will be shown.
 These different paths are described by the following the ``when`` blocks shown
@@ -1705,7 +1705,7 @@ Is referenced parameter is the same group.]]></xs:documentation>
    <xs:annotation>
      <xs:documentation xml:lang="en">This directive describes one potential
 set of input for the tool at this depth. See documentation for the
-[``conditional``](#tool|inputs|conditional) block for more details and examples (XML
+[conditional](#tool-inputs-conditional) block for more details and examples (XML
 and corresponding Cheetah conditionals).</xs:documentation>
    </xs:annotation>
    <xs:sequence>
@@ -1779,7 +1779,7 @@ This is an example test case with multiple repeat elements for the example above
 </test>
 ```

-See the documentation on the [``repeat`` test directive](#tool|tests|test|repeat).
+See the documentation on the [repeat test directive](#tool-tests-test-repeat).

 An older way to specify repeats in a test is by instances that are created by referring to names with a special format: "<repeat name>_<repeat index>|<param name>"

@@ -1904,7 +1904,7 @@ parameter being described. All the attributes for the ``param`` element are
 documented below for completeness, but here are the common ones for each
 type are as follows:

-$attribute_list:name,type,optional,label,help,argument
+$attribute_list:name,type,optional,label,help,argument:4

 ### Parameter Types

@@ -1913,8 +1913,6 @@ $attribute_list:name,type,optional,label,help,argument
 When ``type="text"``, the parameter is free form text and appears as a text box
 in the tool form.

-$attribute_list:value,size,area
-
 ##### Examples

 Sometimes you need labels for data or graph axes, chart titles, etc. This can be
@@ -1932,23 +1930,25 @@ rendered on the tool form as a text area instead of a single line text box.
 <param name="foo" type="text" area="True" size="5x25" />
 ```

+$attribute_list:value,size,area:5
+
 #### ``integer`` and ``float``

 These parameters represent whole number and real numbers, respectively.

-$attribute_list:value,min,max
-
 ##### Example

 ```
 <param name="region_size" size="4" type="integer" value="1" label="flanking regions of size" />
 ```

+$attribute_list:value,min,max:5
+
 #### ``boolean``

 This represents a binary true or false value.

-$attribute_list:checked,truevalue,falsevalue
+$attribute_list:checked,truevalue,falsevalue:5

 #### ``data``

@@ -1998,10 +1998,10 @@ Some example tools using ``multiple="true"`` data parameters include:
 - [multi_data_optional.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/multi_data_optional.xml)

 Additionally, a detailed discussion of handling multiple homogenous files can be found in the
-the [Planemo Documentation](http://planemo.readthedocs.io/en/latest/writing_advanced.html#consuming-collections)
+the [Planemo Documentation](https://planemo.readthedocs.io/en/latest/writing_advanced.html#consuming-collections)
 on this topic.

-$attribute_list:format,multiple
+$attribute_list:format,multiple:5

 #### ``select``

@@ -2027,22 +2027,20 @@ value of ``$upstream_or_down`` will be ``d``, ``u``, ``u,d``, or "".
 </param>
 ```

-$attribute_list:data_ref,dynamic_options,display,multiple
+$attribute_list:data_ref,dynamic_options,display,multiple:5

 #### ``data_column``

 This parameter type is used to select columns from a parameter.

-$attribute_list:force_select,numerical,use_header_name
+$attribute_list:force_select,numerical,use_header_name:5

 #### ``drill_down``

-$attribute_list:hierarchy
+$attribute_list:hierarchy:5

 #### ``data_collection``

-$attribute_list:format,collection_type
-
 The following will create a parameter that only accepts paired FASTQ files grouped into a collection.

 ##### Examples
@@ -2053,12 +2051,12 @@ The following will create a parameter that only accepts paired FASTQ files group
 ```

 More detailed information on writing tools that consume collections can be found
-in the [planemo documentation](http://planemo.readthedocs.io/en/latest/writing_advanced.html#collections).
+in the [planemo documentation](https://planemo.readthedocs.io/en/latest/writing_advanced.html#collections).
+
+$attribute_list:format,collection_type:5

 #### ``color``

-$attribute_list:value,rgb
-
 ##### Examples

 The following example will create a color selector parameter.
@@ -2081,6 +2079,8 @@ sanitizer to prevent Galaxy from escaping the result.
 </param>
 ```

+$attribute_list:value,rgb:5
+
 This covers examples of the most common parameter types, the remaining parameter
 types are more obsecure and less likely to be useful for most tool authors.

@@ -2371,12 +2371,12 @@ parameters supplied in the form with the actual tool executable). Any word
 inside it starting with a dollar sign (``$``) will be treated as a variable whose
 values can be acquired from one of three sources: parameters, metadata, or
 output files. After the substitution of variables with their values, the content
-is interpreted with [Cheetah](http://www.cheetahtemplate.org/) and finally given
+is interpreted with [Cheetah](https://pythonhosted.org/Cheetah/) and finally given
 to the interpreter specified in the corresponding attribute (if any).

 ### Examples

-The following uses a compiled executable ([bedtools](http://bedtools.readthedocs.io/en/latest/)).
+The following uses a compiled executable ([bedtools](https://bedtools.readthedocs.io/en/latest/)).

 ```xml
 <command>bed12ToBed6 -i '$input' > '$output'</command>
@@ -2442,7 +2442,7 @@ the following feature components.

 A set of "metadata information" is defined for each supported data type (see the
 ``MetadataElement`` objects in the various data types classes in
-[``/lib/galaxy/datatypes``](https://github.com/galaxyproject/galaxy/tree/dev/lib/galaxy/datatypes).
+[/lib/galaxy/datatypes](https://github.com/galaxyproject/galaxy/tree/dev/lib/galaxy/datatypes).
 The ``DatasetFilenameWrapper`` class in the
 [/lib/galaxy/tools/wrappers.py](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/tools/wrappers.py)
 code file wraps a metadata collection to return metadata parameters wrapped
@@ -2488,8 +2488,6 @@ In additon to demonstrating accessing metadata, this example demonstrates:
 * ``#set datatype = $input.datatype`` which is the syntax for defining variables
  in Cheetah.

-<a name="cheetah_reserved_variables"></a>
-
 ### Reserved Variables

 Galaxy provides a few pre-defined variables which can be used in your command line,
@@ -2514,7 +2512,7 @@ Name | Description
 ---- | -----------
 ``\${GALAXY_SLOTS:-4}`` | Number of cores/threads allocated by the job runner or resource manager to the tool for the given job (here 4 is the default number of threads to use if running via custom runner that does not configure GALAXY_SLOTS or in an older Galaxy runtime).

-See the [Planemo docs](http://planemo.readthedocs.io/en/latest/writing_advanced.html#cluster-usage)
+See the [Planemo docs](https://planemo.readthedocs.io/en/latest/writing_advanced.html#cluster-usage)
 on the topic of ``GALAXY_SLOTS`` for more information and examples.

 ### Attributes
@@ -2578,7 +2576,7 @@ deprecated and using the ``$__tool_directory__`` variable is superior.
 See [/tools/filters/sorter.xml](https://github.com/galaxyproject/galaxy/blob/master/tools/filters/sorter.xml)
 for typical examples of how to use this tag set. This directive is used to described
 static lists of options and is contained
-within the [``param``](#tool|inputs|param) directive when the ``type`` attribute
+within the [param](#tool-inputs-param) directive when the ``type`` attribute
 value is ``select`` (i.e. ``<param type="select" ...>``).

 ### Example
@@ -2655,7 +2653,7 @@ exclusively use ``filter``s to populate options.
 * ``from_parameter`` - The options for the select list are dynamically obtained
  from a parameter.
 * Using ``filter``s - various filters can be used to populate options, see
-  examples in the [``filter``](#tool|inputs|param|options|filter) documentation.
+  examples in the [filter](#tool-inputs-param-options-filter) documentation.

 ### ``from_data_table``

@@ -2723,7 +2721,7 @@ and demonstrates generating options from a dataset directly.

 Filters can be used to generate options from dataset directly also as the
 example below demonstrates (many more examples are present in the
-[``filter``](#tool|inputs|param|options|filter) documentation).
+[filter](#tool-inputs-param-options-filter) documentation).

 ```xml
 <param name="species1" type="select" label="When Species" multiple="false">
@@ -2741,7 +2739,7 @@ dataset is available, admins must add the database to the local file named
 "blastdb.loc". All such databases in that file are included in the options of
 the select list. For a local instance, the file (e.g. ``blastdb.loc`` or
 ``alignseq.loc``) must be stored in the configured
-[``tool_data_path``](https://github.com/galaxyproject/galaxy/tree/master/tool-data)
+[tool_data_path](https://github.com/galaxyproject/galaxy/tree/master/tool-data)
 directory. In this example, the option names and values are taken from column 0
 of the file.

@@ -2766,7 +2764,7 @@ wrappers for an example of these.

 ### Other Ways to Dynamically Generate Options

-Though deprecated and discouraged, [``code``](#tool|code) blocks can also be
+Though deprecated and discouraged, [code](#tool-code) blocks can also be
 used to generate dynamic options.

 ]]>
@@ -3583,10 +3581,10 @@ This tag set is contained within the ``<outputs>`` tag set, and it defines the
 output dataset collection description resulting from the tool's execution. The
 value of the attribute ``label`` can be acquired from input parameters or
 metadata in the same way that the command line parameters are (discussed in the
-[``command``](#tool|command) directive).
+[command](#tool-command) directive).

 Creating collections in tools is covered in-depth in
-[planemo's documentation](http://planemo.readthedocs.io/en/latest/writing_advanced.html#creating-collections).
+[planemo's documentation](https://planemo.readthedocs.io/en/latest/writing_advanced.html#creating-collections).

 ]]></xs:documentation>
    </xs:annotation>
@@ -3704,7 +3702,7 @@ Galaxy, including:
 * https://github.com/galaxyproject/galaxy/tree/master/test/functional/tools/multi_output_configured.xml

 More information can be found on Planemo's documentation for
-[multiple output files](http://planemo.readthedocs.io/en/latest/writing_advanced.html#multiple-output-files).
+[multiple output files](https://planemo.readthedocs.io/en/latest/writing_advanced.html#multiple-output-files).
 ]]></xs:documentation>
    </xs:annotation>
    <xs:attribute name="pattern" type="xs:string" use="required">
@@ -3964,7 +3962,7 @@ This directive is contained within an output ``data``'s  ``actions`` directive
 describes modifications to either the output's format or metadata (based on
 whether ``type`` is ``format`` or ``metadata``).

-See [``actions``](#tool|outputs|data|actions) documentation for examples
+See [actions](#tool-outputs-data-actions) documentation for examples
 of this directive.

 ]]></xs:documentation>
@@ -4033,7 +4031,7 @@ This directive describes the state of the inputs required to apply an ``action``
 (specified as children of the child ``when`` directives to this element) to an
 output.

-See [``actions``](#tool|outputs|data|actions) documentation for examples
+See [actions](#tool-outputs-data-actions) documentation for examples
 of this directive.

 ]]></xs:documentation>
@@ -4053,7 +4051,7 @@ conditional logic on. The value of this parameter will be matched against nested
    <xs:annotation>
      <xs:documentation xml:lang="en"><![CDATA[

-See [``actions``](#tool|outputs|data|actions) documentation for examples
+See [actions](#tool-outputs-data-actions) documentation for examples
 of this directive.

      ]]></xs:documentation>
@@ -4345,7 +4343,7 @@ response to this directive.
 order to get the tool's version string. The resulting value will be found in the
 "Info" field of the history dataset.

-Unlike the [``command``](#tool|command) tag, with the exception of the string
+Unlike the [command](#tool-command) tag, with the exception of the string
 ``$__tool_directory__`` this value is taken as a literal and so there is no
 need to escape values like ``$`` and command inputs are not available for variable
 substitution.
@@ -4369,7 +4367,7 @@ the tool might be:
 Examples are included in the test tools directory including:

 - [version_command_plain.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/version_command_plain.xml)
- [version_command_tool_dir.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/version_tool_dir.xml)
+- [version_command_tool_dir.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/version_command_tool_dir.xml)
 - [version_command_interpreter.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/version_command_interpreter.xml) (*deprecated*)

 ]]></xs:documentation>
@@ -4599,7 +4597,7 @@ Tools may use exit codes to indicate specific execution errors. Many programs us

 The exit code's range can be any consecutive group of integers. More advanced ranges, such as noncontiguous ranges, are currently not supported. Ranges can be specified in the form "m:n", where m is the start integer and n is the end integer. If ":n" is specified, then the exit code will be compared against all integers less than or equal to n. If "m:" is used, then the exit code will be compared against all integers greater than or equal to m. If the exit code matches, then the error level is applied and the error's description is added to stderr. If a tool's exit code does not match any of the supplied <exit_code> tags' ranges, then no errors are applied to the tool's execution.

-Note that most Unix and Linux variants only support positive integers 0 to 255 for exit codes. If an exit code falls out of the range 0 to 255, the usual convention is to only use the lower 8 bits for the exit code. The only known exception is if a job is broken into subtasks using the tasks runner and one of those tasks is stopped with a POSIX signal. (Note that signals should be used as a last resort for terminating processes.) In those cases, the task will receive -1 times the signal number. For example, suppose that a job uses the tasks runner and 8 tasks are created for the job. If one of the tasks hangs, then a sysadmin may choose to send the "kill" signal, SIGKILL, to the process. In that case, the task (and its job) will exit with an exit code of -9. More on POSIX signals can be found at http://en.wikipedia.org/wiki/Unix_signal as well as man pages on "signal".
+Note that most Unix and Linux variants only support positive integers 0 to 255 for exit codes. If an exit code falls out of the range 0 to 255, the usual convention is to only use the lower 8 bits for the exit code. The only known exception is if a job is broken into subtasks using the tasks runner and one of those tasks is stopped with a POSIX signal. (Note that signals should be used as a last resort for terminating processes.) In those cases, the task will receive -1 times the signal number. For example, suppose that a job uses the tasks runner and 8 tasks are created for the job. If one of the tasks hangs, then a sysadmin may choose to send the "kill" signal, SIGKILL, to the process. In that case, the task (and its job) will exit with an exit code of -9. More on POSIX signals can be found at https://en.wikipedia.org/wiki/Unix_signal as well as man pages on "signal".

 The <exit_code> tag's supported attributes are as follows:

@@ -4845,7 +4843,7 @@ well. Likewise, the history menu includes an option allowing users to aggregate
 all such citations across an analysis in a list of citations.

 BibTeX entries for citations annotated with DOIs will be fetched by Galaxy from
-http://dx.doi.org/ and cached.
+https://dx.doi.org/ and cached.

 ```xml
 <citations>
@@ -109,7 +109,8 @@ class SimpleGraph( object ):

    def gen_edge_dicts( self ):
        """
-        Returns a generator that yields node dictionaries in the form:
+        Returns a generator that yields node dictionaries in the form::
+
            {
                'source': <the index of the source node in the graph's node list>,
                'target': <the index of the target node in the graph's node list>,
@@ -121,7 +122,8 @@ class SimpleGraph( object ):

    def as_dict( self ):
        """
-        Returns a dictionary of the form
+        Returns a dictionary of the form::
+
            { 'nodes': <a list of node dictionaries>, 'edges': <a list of node dictionaries> }
        """
        return { 'nodes': list( self.gen_node_dicts() ), 'edges': list( self.gen_edge_dicts() ) }
@@ -619,7 +619,7 @@ class HistoryContentsController( BaseAPIController, UsesLibraryMixin, UsesLibrar

        :returns:   archive file for download

-        .. note: this is a volatile endpoint and settings and behavior may change.
+        .. note:: this is a volatile endpoint and settings and behavior may change.
        """
        # roughly from: http://stackoverflow.com/a/31976060 (windows, linux)
        invalid_filename_char_regex = re.compile( r'[:<>|\\\/\?\* "]' )
@@ -146,15 +146,15 @@ class Download( object ):

    def url_download( self, install_dir, downloaded_file_name, download_url, extract=True, checksums={} ):
        """
-            The given download_url can have an extension like #md5#, #sha256#, (or #md5= to support pypi defaults).
+        The given download_url can have an extension like #md5#, #sha256#, (or #md5= to support pypi defaults).

-                https://pypi.python.org/packages/source/k/khmer/khmer-1.0.tar.gz#md5#b60639a8b2939836f66495b9a88df757
+            https://pypi.python.org/packages/source/k/khmer/khmer-1.0.tar.gz#md5#b60639a8b2939836f66495b9a88df757

-            Alternatively, to not break HTTP spec, you can specify md5 and
-            sha256 as keys in the <action /> element.
+        Alternatively, to not break HTTP spec, you can specify md5 and
+        sha256 as keys in the <action /> element.

-            This indicates a checksum which will be checked after download.
-            If the checksum does not match an exception is thrown.
+        This indicates a checksum which will be checked after download.
+        If the checksum does not match an exception is thrown.
        """
        file_path = os.path.join( install_dir, downloaded_file_name )
        src = None
@@ -300,7 +300,8 @@ class AssertDirectoryExists( RecipeStep ):
    def assert_directory_exists( self, full_path ):
        """
        Return True if a symbolic link or directory exists, but if full_path is a file,
-        return False.    """
+        return False.
+        """
        if full_path is None:
            return False
        if os.path.isfile( full_path ):
@@ -911,20 +912,20 @@ class RegexReplace( RecipeStep ):
        filtered_actions and dir.

        This step supports the full range of python's regular expression engine, including backreferences in
-        the replacement text.
+        the replacement text. Example::

-        Example:
-        <action type="regex_replace" filename="Makefile">
-            <regex>^CFLAGS(\s*)=\s*-g\s*-Wall\s*-O2\s*$$</regex>
-            <replacement>CFLAGS\1= -g -Wall -O2 -I$$(NCURSES_INCLUDE_PATH)/ncurses/ -I$$(NCURSES_INCLUDE_PATH) -L$$(NCURSES_LIB_PATH)</replacement>
-        </action>
+            <action type="regex_replace" filename="Makefile">
+                <regex>^CFLAGS(\s*)=\s*-g\s*-Wall\s*-O2\s*$$</regex>
+                <replacement>CFLAGS\1= -g -Wall -O2 -I$$(NCURSES_INCLUDE_PATH)/ncurses/ -I$$(NCURSES_INCLUDE_PATH) -L$$(NCURSES_LIB_PATH)</replacement>
+            </action>

-        Before:
-        CFLAGS  = -g -Wall -O2
+        Before::

-        After:
-        CFLAGS  = -g -Wall -O2 -I$(NCURSES_INCLUDE_PATH)/ncurses/ -I$(NCURSES_INCLUDE_PATH) -L$(NCURSES_LIB_PATH)
+            CFLAGS  = -g -Wall -O2

+        After::
+
+            CFLAGS  = -g -Wall -O2 -I$(NCURSES_INCLUDE_PATH)/ncurses/ -I$(NCURSES_INCLUDE_PATH) -L$(NCURSES_LIB_PATH)
        """
        log_file = os.path.join( install_environment.install_dir, basic_util.INSTALLATION_LOG )
        if os.path.exists( log_file ):
@@ -1011,20 +1012,19 @@ class SetEnvironment( RecipeStep ):

        The first tag set defines a complex repository dependency like this.  This tag set ensures that changeset
        revision XXX of the repository named package_graphicsmagick_1_3 owned by YYY in the tool shed ZZZ has been
-        previously installed.
+        previously installed::

-        <tool_dependency>
-            <package name="graphicsmagick" version="1.3.18">
-                <repository changeset_revision="XXX" name="package_graphicsmagick_1_3" owner="YYY" prior_installation_required="True" toolshed="ZZZ" />
-            </package>
-            ...
+            <tool_dependency>
+                <package name="graphicsmagick" version="1.3.18">
+                    <repository changeset_revision="XXX" name="package_graphicsmagick_1_3" owner="YYY" prior_installation_required="True" toolshed="ZZZ" />
+                </package>
+                ...

-        * By the way, there is an env.sh file associated with version 1.3.18 of the graphicsmagick package which looks
-        something like this (we'll reference this file later in this discussion.
-        ----
-        GRAPHICSMAGICK_ROOT_DIR=/<my configured tool dependency path>/graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick;
-        export GRAPHICSMAGICK_ROOT_DIR
-        ----
+        By the way, there is an env.sh file associated with version 1.3.18 of the graphicsmagick package which looks
+        something like this (we'll reference this file later in this discussion)::
+
+            GRAPHICSMAGICK_ROOT_DIR=/<my configured tool dependency path>/graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick;
+            export GRAPHICSMAGICK_ROOT_DIR

        The second tag set defines a specific package dependency that has been previously installed (guaranteed by the
        tag set discussed above) and compiled, where the compiled dependency is needed by the tool dependency currently
@@ -1033,44 +1033,42 @@ class SetEnvironment( RecipeStep ):
        version 2.0.0 of the osra package requires version 1.3.18 of the graphicsmagick package in order to successfully
        compile.  When this tag set is handled, one of the effects is that the env.sh file associated with graphicsmagick
        version 1.3.18 is "sourced", which undoubtedly sets or alters certain environment variables (e.g. PATH, PYTHONPATH,
-        etc).
+        etc)::

-        <!-- populate the environment variables from the dependent repositories -->
-        <action type="set_environment_for_install">
-            <repository changeset_revision="XXX" name="package_graphicsmagick_1_3" owner="YYY" toolshed="ZZZ">
-                <package name="graphicsmagick" version="1.3.18" />
-            </repository>
-        </action>
+            <!-- populate the environment variables from the dependent repositories -->
+            <action type="set_environment_for_install">
+                <repository changeset_revision="XXX" name="package_graphicsmagick_1_3" owner="YYY" toolshed="ZZZ">
+                    <package name="graphicsmagick" version="1.3.18" />
+                </repository>
+            </action>

        The third tag set enables discovery of the same required package dependency discussed above for correctly compiling
        the osra version 2.0.0 package, but in this case the package can be discovered at tool execution time.  Using the
        $ENV[] option as shown in this example, the value of the environment variable named GRAPHICSMAGICK_ROOT_DIR (which
        was set in the environment using the second tag set described above) will be used to automatically alter the env.sh
        file associated with the osra version 2.0.0 tool dependency when it is installed into Galaxy.  * Refer to where we
-        discussed the env.sh file for version 1.3.18 of the graphicsmagick package above.
+        discussed the env.sh file for version 1.3.18 of the graphicsmagick package above::

-        <action type="set_environment">
-            <environment_variable action="prepend_to" name="LD_LIBRARY_PATH">$ENV[GRAPHICSMAGICK_ROOT_DIR]/lib/</environment_variable>
-            <environment_variable action="prepend_to" name="LD_LIBRARY_PATH">$INSTALL_DIR/potrace/build/lib/</environment_variable>
-            <environment_variable action="prepend_to" name="PATH">$INSTALL_DIR/bin</environment_variable>
-            <!-- OSRA_DATA_FILES is only used by the galaxy wrapper and is not part of OSRA -->
-            <environment_variable action="set_to" name="OSRA_DATA_FILES">$INSTALL_DIR/share</environment_variable>
-        </action>
+            <action type="set_environment">
+                <environment_variable action="prepend_to" name="LD_LIBRARY_PATH">$ENV[GRAPHICSMAGICK_ROOT_DIR]/lib/</environment_variable>
+                <environment_variable action="prepend_to" name="LD_LIBRARY_PATH">$INSTALL_DIR/potrace/build/lib/</environment_variable>
+                <environment_variable action="prepend_to" name="PATH">$INSTALL_DIR/bin</environment_variable>
+                <!-- OSRA_DATA_FILES is only used by the galaxy wrapper and is not part of OSRA -->
+                <environment_variable action="set_to" name="OSRA_DATA_FILES">$INSTALL_DIR/share</environment_variable>
+            </action>

        The above tag will produce an env.sh file for version 2.0.0 of the osra package when it it installed into Galaxy
        that looks something like this.  Notice that the path to the gmagick binary is included here since it expands the
-        defined $ENV[GRAPHICSMAGICK_ROOT_DIR] value in the above tag set.
+        defined $ENV[GRAPHICSMAGICK_ROOT_DIR] value in the above tag set::

-        ----
-        LD_LIBRARY_PATH=/<my configured tool dependency path>/graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick/lib/:$LD_LIBRARY_PATH;
-        export LD_LIBRARY_PATH
-        LD_LIBRARY_PATH=/<my configured tool dependency path>/osra/1.4.0/YYY/depends_on/XXX/potrace/build/lib/:$LD_LIBRARY_PATH;
-        export LD_LIBRARY_PATH
-        PATH=/<my configured tool dependency path>/osra/1.4.0/YYY/depends_on/XXX/bin:$PATH;
-        export PATH
-        OSRA_DATA_FILES=/<my configured tool dependency path>/osra/1.4.0/YYY/depends_on/XXX/share;
-        export OSRA_DATA_FILES
-        ----
+            LD_LIBRARY_PATH=/<my configured tool dependency path>/graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick/lib/:$LD_LIBRARY_PATH;
+            export LD_LIBRARY_PATH
+            LD_LIBRARY_PATH=/<my configured tool dependency path>/osra/1.4.0/YYY/depends_on/XXX/potrace/build/lib/:$LD_LIBRARY_PATH;
+            export LD_LIBRARY_PATH
+            PATH=/<my configured tool dependency path>/osra/1.4.0/YYY/depends_on/XXX/bin:$PATH;
+            export PATH
+            OSRA_DATA_FILES=/<my configured tool dependency path>/osra/1.4.0/YYY/depends_on/XXX/share;
+            export OSRA_DATA_FILES
        """
        env_var_value = env_var_dict[ 'value' ]
        # env_var_value is the text of an environment variable tag like this:
@@ -1575,7 +1573,6 @@ class SetupPythonEnvironment( Download, RecipeStep ):
        although it may never be necessary.  If initial_download is True, the recipe steps will be filtered
        and returned and the installation directory (i.e., dir) will be defined and returned.  If we're not
        in the initial download stage, these actions will not occur, and None values will be returned for them.
-
        """
        # <action type="setup_python_environment">
        #       <repository name="package_python_2_7" owner="bgruening">
@@ -46,9 +46,8 @@ DEVTEAM = [
 TEMPLATE = """
 .. to_doc

-------------------------------
 %s
-------------------------------
+===============================

 .. announce_start

@@ -1,56 +0,0 @@
-===========================================================
-TODO Galaxy Release (v %s)
-===========================================================
-
-.. include:: _header.rst
-
-Highlights
-===========================================================
-
-**Feature1**
-  Feature description.
-
-**Feature2**
-  Feature description.
-
-**Feature3**
-  Feature description.
-
-`Github <https://github.com/galaxyproject/galaxy>`__
-===========================================================
-
-New
-  .. code-block:: shell
-  
-      % git clone -b master https://github.com/galaxyproject/galaxy.git
-
-Update to latest stable release
-  .. code-block:: shell
-  
-      % git checkout master && pull --ff-only origin master
-
-Update to exact version
-  .. code-block:: shell
-  
-      % git checkout v%s
-
-
-`BitBucket <https://bitbucket.org/galaxy/galaxy-dist>`__
-===========================================================
-
-Upgrade
-  .. code-block:: shell
-  
-      % hg pull
-      % hg update latest_%s
-
-
-See `our wiki <https://wiki.galaxyproject.org/Develop/SourceCode>`__ for additional details regarding the source code locations.
-
-Release Notes
-===========================================================
-
-.. include:: %s.rst
-   :start-after: enhancements
-
-.. include:: _thanks.rst
@@ -57,11 +57,8 @@ REQUIRES_DAEMONIZE_MESSAGE = "Attempted to use Galaxy in daemon mode, but daemon

 log = logging.getLogger(__name__)

-base_file = os.path.basename(__file__)
-if base_file == 'main.py':
-    GALAXY_ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
-elif base_file == 'galaxy-main':
-    GALAXY_ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+real_file = os.path.realpath(__file__)
+GALAXY_ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(real_file), os.pardir))
 GALAXY_LIB_DIR = os.path.join(GALAXY_ROOT_DIR, "lib")
 DEFAULT_INI_APP = "main"
 DEFAULT_INIS = ["config/galaxy.ini", "universe_wsgi.ini", "config/galaxy.ini.sample"]