From a8bc9d5d16244badb023a9c5097efd826b26db56 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 18 Sep 2016 13:56:35 -0400 Subject: [PATCH] XSD Generated Tool Documentation. - Add script to generate Markdown documentation from the XSD. - Add Makefile target to build these docs, convert to HTML, and open respectively. --- .gitignore | 2 + Makefile | 16 +++- doc/parse_gx_xsd.py | 201 +++++++++++++++++++++++++++++++++++++++++ doc/schema_template.md | 79 ++++++++++++++++ 4 files changed, 296 insertions(+), 2 deletions(-) create mode 100644 doc/parse_gx_xsd.py create mode 100644 doc/schema_template.md diff --git a/.gitignore b/.gitignore index 214603d00d8..6fb1cacce1f 100644 --- a/.gitignore +++ b/.gitignore @@ -124,6 +124,8 @@ bower_components # Documentation build files. doc/build +doc/schema.html +doc/schema.md # Misc *.orig diff --git a/Makefile b/Makefile index 789680cfcbd..5049d7f3c9f 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,9 @@ IN_VENV=if [ -f $(VENV)/bin/activate ]; then . $(VENV)/bin/activate; fi; PROJECT_URL?=https://github.com/galaxyproject/galaxy GRUNT_DOCKER_NAME:=galaxy/client-builder:16.01 GRUNT_EXEC?=node_modules/grunt-cli/bin/grunt +DOCS_DIR=doc +OPEN_RESOURCE=bash -c 'open $$0 || xdg-open $$0' + all: help @echo "This makefile is primarily used for building Galaxy's JS client. A sensible all target is not yet implemented." @@ -21,13 +24,22 @@ docs: ## generate Sphinx HTML documentation, including API docs $(IN_VENV) $(MAKE) -C doc clean $(IN_VENV) $(MAKE) -C doc html +docs-schema-ready: ## Build Github-flavored Markdown from Galaxy Tool XSD + python $(DOCS_DIR)/parse_gx_xsd.py > $(DOCS_DIR)/schema.md + +docs-schema-html: docs-schema-ready ## Convert Galaxy Tool XSD Markdown docs into HTML + markdown $(DOCS_DIR)/schema.md > $(DOCS_DIR)/schema.html + +open-docs-schema: docs-schema-html ## Open HTML generated from Galaxy Tool XSD. + $(OPEN_RESOURCE) $(DOCS_DIR)/schema.html + _open-docs: - open doc/_build/html/index.html || xdg-open doc/_build/html/index.html + $(OPEN_RESOURCE) $(DOCS_DIR)/_build/html/index.html open-docs: docs _open-docs ## generate Sphinx HTML documentation and open in browser open-project: ## open project on github - open $(PROJECT_URL) || xdg-open $(PROJECT_URL) + $(OPEN_RESOURCE) $(PROJECT_URL) lint: ## check style using tox and flake8 for Python 2 and Python 3 $(IN_VENV) tox -e py27-lint && tox -e py34-lint diff --git a/doc/parse_gx_xsd.py b/doc/parse_gx_xsd.py new file mode 100644 index 00000000000..7263ae358fd --- /dev/null +++ b/doc/parse_gx_xsd.py @@ -0,0 +1,201 @@ +# coding: utf-8 +# TODO: Add examples, tables and best practice links to command +# TODO: Examples of truevalue, falsevalue +# TODO: Test param extra_file +from __future__ import print_function + +# Things dropped from TOC (still documented inside schema). +# - request_parameter_translation + +from lxml import etree +from StringIO import StringIO + + +with open("doc/schema_template.md", "r") as f: + MARKDOWN_TEMPLATE = f.read() + +with open("lib/galaxy/tools/xsd/galaxy.xsd", "r") as f: + xmlschema_doc = etree.parse(f) + +markdown_buffer = StringIO() + + +def main(): + """Entry point for the function that builds Markdown help for the Galaxy XSD.""" + for line in MARKDOWN_TEMPLATE.splitlines(): + if line.startswith("$tag:"): + print(Tag(line).build_help()) + elif line.startswith("$toc"): + print_toc() + else: + print(line) + + +def print_toc(): + tags = [] + for line in MARKDOWN_TEMPLATE.splitlines(): + if line.startswith("$tag:"): + tags.append(Tag(line)) + + for i, tag in enumerate(tags): + try: + next_tag = tags[i + 1] + next_count = next_tag.title.count("|") + except IndexError: + next_tag = None + next_count = None + line = "" + count = tag.title.count("|") + for c in range(count): + if next_count is not None and c < next_count: + if c == 0: + line += "├──" + else: + line += "┼──" + else: + if c == 0: + line += "└──" + else: + line += "┴──" + line += "[``<" + tag.title.split("|")[-1] + ">``](#" + tag.title + ") " + print(line) + + +class Tag(object): + + def __init__(self, line): + assert line.startswith("$tag:") + line_parts = line.split(" ") + first_part = line_parts[0] + hide_attributes = False + if len(line_parts) > 1: + if "hide_attributes" in line_parts[1]: + hide_attributes = True + _, title, xpath = first_part.split(":") + xpath = xpath.replace("/element", "/{http://www.w3.org/2001/XMLSchema}element") + xpath = xpath.replace("/group", "/{http://www.w3.org/2001/XMLSchema}group") + xpath = xpath.replace("/complexType", "/{http://www.w3.org/2001/XMLSchema}complexType") + self.xpath = xpath + self.hide_attributes = hide_attributes + self.title = title + + def build_help(self): + tag = xmlschema_doc.find(self.xpath) + if tag is None: + raise Exception("Could not find xpath for %s" % self.xpath) + + title = self.title + tag_help = StringIO() + tag_help.write("""\n\n""" % title) + tag_help.write("## " + " > ".join(["``%s``" % p for p in title.split("|")])) + tag_help.write("\n") + tag_help.write(_build_tag(tag, self.hide_attributes)) + tag_help.write("\n\n") + return tag_help.getvalue() + + +def _build_tag(tag, hide_attributes): + tag_el = _find_tag_el(tag) + attributes = _find_attributes(tag) + tag_help = StringIO() + annotation_el = tag_el.find("{http://www.w3.org/2001/XMLSchema}annotation") + text = annotation_el.find("{http://www.w3.org/2001/XMLSchema}documentation").text + for line in text.splitlines(): + if line.startswith("$attribute_list:"): + attributes_str = line.split(":", 1)[1] + attribute_names = attributes_str.split(",") + text = text.replace(line, _build_attributes_table(tag, attributes, attribute_names=attribute_names)) + if line.startswith("$assertions"): + assertions_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}complexType[@name='TestAssertions']") + assertion_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}group[@name='TestAssertion']") + assertions_buffer = StringIO() + assertions_buffer.write(_doc_or_none(assertions_tag)) + assertions_buffer.write("\n\n") + assertions_buffer.write("Child Element/Assertion | Details \n") + assertions_buffer.write("--- | ---\n") + elements = assertion_tag.findall("{http://www.w3.org/2001/XMLSchema}choice/{http://www.w3.org/2001/XMLSchema}element") + for element in elements: + doc = _doc_or_none(element).strip() + assertions_buffer.write("``%s`` | %s\n" % (element.attrib["name"], doc)) + text = text.replace(line, assertions_buffer.getvalue()) + tag_help.write(text) + best_practices = _get_bp_link(annotation_el) + if best_practices: + tag_help.write("\n\n### Best Practices\n") + tag_help.write(""" +Find the Intergalactic Utilities Commision suggested best practices for this +element [here](%s).""" % best_practices) + tag_help.write(_build_attributes_table(tag, attributes, hide_attributes)) + + return tag_help.getvalue() + + +def _get_bp_link(annotation_el): + anchor = annotation_el.attrib.get("{http://galaxyproject.org/xml/1.0}best_practices", None) + link = None + if anchor: + link = "http://planemo.readthedocs.io/en/latest/standards/docs/best_practices/tool_xml.html#%s" % anchor + return link + + +def _build_attributes_table(tag, attributes, hide_attributes=False, attribute_names=None): + attribute_table = StringIO() + attribute_table.write("\n\n") + if attributes and not hide_attributes: + attribute_table.write("\n### Attributes\n") + attribute_table.write("Attribute | Details | Required\n") + attribute_table.write("--- | --- | ---\n") + for attribute in attributes: + name = attribute.attrib["name"] + if attribute_names and name not in attribute_names: + continue + details = _doc_or_none(attribute) + if details is None: + type_el = _type_el(attribute) + details = _doc_or_none(type_el) + annotation_el = type_el.find("{http://www.w3.org/2001/XMLSchema}annotation") + else: + annotation_el = attribute.find("{http://www.w3.org/2001/XMLSchema}annotation") + + use = attribute.attrib.get("use", "optional") == "required" + if "|" in details: + raise Exception("Cannot build Markdown table") + details = details.replace("\n", " ").strip() + best_practices = _get_bp_link(annotation_el) + if best_practices: + details += """ Find the Intergalactic Utilities Commision suggested best practices for this element [here](%s).""" % best_practices + + attribute_table.write("``%s`` | %s | %s\n" % (name, details, use)) + return attribute_table.getvalue() + + +def _find_attributes(tag): + return tag.findall("{http://www.w3.org/2001/XMLSchema}attribute") or \ + tag.findall("{http://www.w3.org/2001/XMLSchema}complexType/{http://www.w3.org/2001/XMLSchema}attribute") or \ + tag.findall("{http://www.w3.org/2001/XMLSchema}complexContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attribute") or \ + tag.findall("{http://www.w3.org/2001/XMLSchema}simpleContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attribute") + + +def _find_tag_el(tag): + if _doc_or_none(tag) is not None: + return tag + + return _type_el(tag) + + +def _type_el(tag): + element_type = tag.attrib["type"] + type_el = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}complexType/[@name='%s']" % element_type) or \ + xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}simpleType/[@name='%s']" % element_type) + return type_el + + +def _doc_or_none(tag): + doc_el = tag.find("{http://www.w3.org/2001/XMLSchema}annotation/{http://www.w3.org/2001/XMLSchema}documentation") + if doc_el is None: + return None + else: + return doc_el.text + +if __name__ == '__main__': + main() diff --git a/doc/schema_template.md b/doc/schema_template.md new file mode 100644 index 00000000000..4277482e548 --- /dev/null +++ b/doc/schema_template.md @@ -0,0 +1,79 @@ +# Galaxy Tool XML File + +The XML File for a Galaxy tool, generally referred to as the "tool config +file" or "wrapper", serves a number of purposes. First, it lays out the user +interface for the tool ( e.g. form fields, text, help, etc...). Second, it +provides the glue that links your tool to Galaxy by telling Galaxy how to +invoke it, what options to pass, and what files it will produce as output. + +This document serves as reference documentation. If you would like to learn +how to build tools for Galaxy, +[Planemo](http://planemo.readthedocs.io/en/latest/writing.html) features a +number of tutorials on building Galaxy tools that would better serve that purpose. + +$toc + +$tag:tool://element[@name='tool'] +$tag:tool|description://element[@name='tool']//element[@name='description'] +$tag:tool|version_command://element[@name='tool']//element[@name='version_command'] +$tag:tool|command://element[@name='tool']//element[@name='command'] hide_attributes +$tag:tool|inputs://complexType[@name='Inputs'] +$tag:tool|inputs|section://complexType[@name='Section'] +$tag:tool|inputs|repeat://complexType[@name='Repeat'] +$tag:tool|inputs|conditional://complexType[@name='Conditional'] +$tag:tool|inputs|conditional|when://complexType[@name='ConditionalWhen'] +$tag:tool|inputs|param://complexType[@name='Param'] +$tag:tool|inputs|param|validator://complexType[@name='Validator'] +$tag:tool|inputs|param|option://complexType[@name='ParamOption'] +$tag:tool|inputs|param|options://complexType[@name='ParamOptions'] +$tag:tool|inputs|param|options|column://complexType[@name='Column'] +$tag:tool|inputs|param|options|filter://complexType[@name='Filter'] +$tag:tool|inputs|param|sanitizer://complexType[@name='Sanitizer'] +$tag:tool|inputs|param|sanitizer|valid://complexType[@name='SanitizerValid'] +$tag:tool|inputs|param|sanitizer|valid|add://complexType[@name='SanitizerValidAdd'] +$tag:tool|inputs|param|sanitizer|valid|remove://complexType[@name='SanitizerValidRemove'] +$tag:tool|inputs|param|sanitizer|mapping://complexType[@name='SanitizerMapping'] +$tag:tool|inputs|param|sanitizer|mapping|add://complexType[@name='SanitizerMappingAdd'] +$tag:tool|inputs|param|sanitizer|mapping|remove://complexType[@name='SanitizerMappingRemove'] +$tag:tool|configfiles://complexType[@name='ConfigFiles'] +$tag:tool|configfiles|configfile://complexType[@name='ConfigFile'] +$tag:tool|configfiles|inputs://complexType[@name='ConfigInputs'] +$tag:tool|environment_variables://complexType[@name='EnvironmentVariables'] +$tag:tool|environment_variables|environment_variable://complexType[@name='EnvironmentVariable'] +$tag:tool|outputs://complexType[@name='Outputs'] +$tag:tool|outputs|data://complexType[@name='Data'] +$tag:tool|outputs|data|filter://complexType[@name='OutputFilter'] +$tag:tool|outputs|data|change_format://complexType[@name='ChangeFormat'] +$tag:tool|outputs|data|change_format|when://complexType[@name='ChangeFormatWhen'] +$tag:tool|outputs|data|actions://complexType[@name='Actions'] +$tag:tool|outputs|data|actions|conditional://complexType[@name='ActionsConditional'] +$tag:tool|outputs|data|actions|conditional|when://complexType[@name='ActionsConditionalWhen'] +$tag:tool|outputs|data|actions|action://complexType[@name='Action'] +$tag:tool|outputs|data|discover_datasets://complexType[@name='OutputDiscoverDatasets'] +$tag:tool|outputs|collection://complexType[@name='Collection'] +$tag:tool|outputs|collection|filter://complexType[@name='OutputFilter'] +$tag:tool|outputs|collection|discover_datasets://complexType[@name='OutputCollectionDiscoverDatasets'] +$tag:tool|tests://complexType[@name='Tests'] +$tag:tool|tests|test://complexType[@name='Test'] +$tag:tool|tests|test|param://complexType[@name='TestParam'] +$tag:tool|tests|test|repeat://complexType[@name='TestRepeat'] +$tag:tool|tests|test|section://complexType[@name='TestSection'] +$tag:tool|tests|test|conditional://complexType[@name='TestConditional'] +$tag:tool|tests|test|output://complexType[@name='TestOutput'] +$tag:tool|tests|test|output|discover_dataset://complexType[@name='TestDiscoveredDataset'] +$tag:tool|tests|test|output|metadata://complexType[@name='TestOutputMetadata'] +$tag:tool|tests|test|output|assert_contents://group[@name='TestOutputElement']//element[@name='assert_contents'] +$tag:tool|tests|test|output_collection://complexType[@name='TestOutputCollection'] +$tag:tool|tests|test|assert_command://group[@name='TestParamElement']//element[@name='assert_command'] +$tag:tool|tests|test|assert_stdout://group[@name='TestParamElement']//element[@name='assert_stdout'] +$tag:tool|tests|test|assert_stderr://group[@name='TestParamElement']//element[@name='assert_stderr'] +$tag:tool|code://complexType[@name='Code'] +$tag:tool|requirements://complexType[@name='Requirements'] +$tag:tool|requirements|requirement://complexType[@name='Requirement'] +$tag:tool|requirements|container://complexType[@name='Container'] +$tag:tool|stdio://complexType[@name='Stdio'] +$tag:tool|stdio|exit_code://complexType[@name='ExitCode'] hide_attributes +$tag:tool|stdio|regex://complexType[@name='Regex'] hide_attributes +$tag:tool|help://element[@name='tool']//element[@name='help'] +$tag:tool|citations://complexType[@name='Citations'] +$tag:tool|citations|citation://complexType[@name='Citation']