diff --git a/.ci/flake8_blacklist.txt b/.ci/flake8_blacklist.txt index bfef6f7adeb..4f2201f803e 100644 --- a/.ci/flake8_blacklist.txt +++ b/.ci/flake8_blacklist.txt @@ -2,7 +2,6 @@ .tox/ client/node_modules/ database/ -doc/patch.py doc/source/conf.py eggs/ lib/galaxy/util/jstree.py diff --git a/.ci/flake8_lint_include_list.txt b/.ci/flake8_lint_include_list.txt index 5d2ef126439..6799dff7f78 100644 --- a/.ci/flake8_lint_include_list.txt +++ b/.ci/flake8_lint_include_list.txt @@ -5,6 +5,7 @@ cron/cleanup_datasets.py cron/parse_builds_3_sites.py cron/parse_builds.py doc/parse_gx_xsd.py +doc/patch.py lib/galaxy/actions/ lib/galaxy/auth/ lib/galaxy/config.py diff --git a/.ci/py3_sources.txt b/.ci/py3_sources.txt index 1dc46fc4c6d..a85b4fcfcd8 100644 --- a/.ci/py3_sources.txt +++ b/.ci/py3_sources.txt @@ -2,6 +2,7 @@ client/galaxy/style/source_material/circle.py contrib/ cron/ doc/parse_gx_xsd.py +doc/patch.py lib/galaxy/actions/ lib/galaxy/auth/ lib/galaxy/config.py diff --git a/.gitignore b/.gitignore index 3cbb346575a..5d9869fb9e9 100644 --- a/.gitignore +++ b/.gitignore @@ -127,8 +127,8 @@ bower_components # Documentation build files. doc/build -doc/schema.html doc/schema.md +doc/source/dev/schema.rst # Misc *.orig diff --git a/Makefile b/Makefile index 0c1f104516f..d616ca38252 100644 --- a/Makefile +++ b/Makefile @@ -34,15 +34,6 @@ docs-slides-ready: docs-slides-export: docs-slides-ready $(SLIDESHOW_TO_PDF) $(SLIDESHOW_DIR)/galaxy_architecture/galaxy_architecture.html -docs-schema-ready: ## Build Github-flavored Markdown from Galaxy Tool XSD (expects libxml in environment) - python $(DOCS_DIR)/parse_gx_xsd.py > $(DOCS_DIR)/schema.md - -docs-schema-html: docs-schema-ready ## Convert Galaxy Tool XSD Markdown docs into HTML (expects pandoc in environment) - pandoc $(DOCS_DIR)/schema.md -f markdown_github -s -o $(DOCS_DIR)/schema.html - -open-docs-schema: docs-schema-html ## Open HTML generated from Galaxy Tool XSD. - $(OPEN_RESOURCE) $(DOCS_DIR)/schema.html - _open-docs: $(OPEN_RESOURCE) $(DOCS_DIR)/_build/html/index.html diff --git a/doc/Makefile b/doc/Makefile index 49a5cda4720..323da3c2fa2 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -17,8 +17,9 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) sou # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext updaterst +GENERATED_RST = source/dev/schema.rst +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext updaterst help: @echo "Please use \`make ' where is one of" @@ -43,44 +44,49 @@ help: @echo " doctest to run all doctests embedded in the documentation (if enabled)" @echo " updaterst to update sphinx rst to reflect code structure changes" +schema.md: parse_gx_xsd.py schema_template.md ../lib/galaxy/tools/xsd/galaxy.xsd ## Build Github-flavored Markdown from Galaxy Tool XSD (expects libxml in environment) + python parse_gx_xsd.py schema_template.md ../lib/galaxy/tools/xsd/galaxy.xsd > $@ + +source/dev/schema.rst: schema.md ## Convert Galaxy Tool XSD Markdown docs into reStructuredText (expects pandoc in environment) + pandoc schema.md -f markdown_github-hard_line_breaks -s --toc --toc-depth=2 -o $@ # might also want to do # cd source/lib; hg revert; rm *.rst.orig; or not. clean: - -rm -rf $(BUILDDIR)/* $(UPDATEWORKDIR) + -rm -rf $(BUILDDIR)/* $(UPDATEWORKDIR) schema.md $(GENERATED_RST) -html: $(TOOLDATABUILDFILES) +html: $(GENERATED_RST) $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." -dirhtml: +dirhtml: $(GENERATED_RST) $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." -singlehtml: +singlehtml: $(GENERATED_RST) $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." -pickle: +pickle: $(GENERATED_RST) $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." -json: +json: $(GENERATED_RST) $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." -htmlhelp: +htmlhelp: $(GENERATED_RST) $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." -qthelp: +qthelp: $(GENERATED_RST) $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ @@ -89,7 +95,7 @@ qthelp: @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Galaxy.qhc" -devhelp: +devhelp: $(GENERATED_RST) $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @@ -98,30 +104,30 @@ devhelp: @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Galaxy" @echo "# devhelp" -epub: +epub: $(GENERATED_RST) $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." -latex: +latex: $(GENERATED_RST) $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." -latexpdf: +latexpdf: $(GENERATED_RST) $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." -text: +text: $(GENERATED_RST) $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." -man: +man: $(GENERATED_RST) $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." @@ -133,29 +139,29 @@ texinfo: @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." -info: +info: $(GENERATED_RST) $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." -gettext: +gettext: $(GENERATED_RST) $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." -changes: +changes: $(GENERATED_RST) $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." -linkcheck: +linkcheck: $(GENERATED_RST) $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." -doctest: +doctest: $(GENERATED_RST) $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/parse_gx_xsd.py b/doc/parse_gx_xsd.py index 452b7f4e43b..0ff1b034cbe 100644 --- a/doc/parse_gx_xsd.py +++ b/doc/parse_gx_xsd.py @@ -2,17 +2,19 @@ # TODO: Add examples, tables and best practice links to command # TODO: Examples of truevalue, falsevalue # TODO: Test param extra_file -# Things dropped from TOC (still documented inside schema). +# Things dropped from schema_template.md (still documented inside schema). # - request_parameter_translation from __future__ import print_function +import sys + from lxml import etree from six import StringIO -with open("doc/schema_template.md", "r") as f: +with open(sys.argv[1], "r") as f: MARKDOWN_TEMPLATE = f.read() -with open("lib/galaxy/tools/xsd/galaxy.xsd", "r") as f: +with open(sys.argv[2], "r") as f: xmlschema_doc = etree.parse(f) markdown_buffer = StringIO() @@ -23,42 +25,10 @@ def main(): for line in MARKDOWN_TEMPLATE.splitlines(): if line.startswith("$tag:"): print(Tag(line).build_help()) - elif line.startswith("$toc"): - print_toc() else: print(line) -def print_toc(): - tags = [] - for line in MARKDOWN_TEMPLATE.splitlines(): - if line.startswith("$tag:"): - tags.append(Tag(line)) - - for i, tag in enumerate(tags): - try: - next_tag = tags[i + 1] - next_count = next_tag.title.count("|") - except IndexError: - next_tag = None - next_count = None - line = "" - count = tag.title.count("|") - for c in range(count): - if next_count is not None and c < next_count: - if c == 0: - line += "├──" - else: - line += "┼──" - else: - if c == 0: - line += "└──" - else: - line += "┴──" - line += "[``<" + tag.title.split("|")[-1] + ">``](#" + tag.title + ") " - print(line) - - class Tag(object): def __init__(self, line): @@ -84,7 +54,6 @@ class Tag(object): title = self.title tag_help = StringIO() - tag_help.write("""\n\n""" % title) tag_help.write("## " + " > ".join(["``%s``" % p for p in title.split("|")])) tag_help.write("\n") tag_help.write(_build_tag(tag, self.hide_attributes)) @@ -100,9 +69,10 @@ def _build_tag(tag, hide_attributes): text = annotation_el.find("{http://www.w3.org/2001/XMLSchema}documentation").text for line in text.splitlines(): if line.startswith("$attribute_list:"): - attributes_str = line.split(":", 1)[1] + attributes_str, header_level = line.split(":")[1:3] attribute_names = attributes_str.split(",") - text = text.replace(line, _build_attributes_table(tag, attributes, attribute_names=attribute_names)) + header_level = int(header_level) + text = text.replace(line, _build_attributes_table(tag, attributes, attribute_names=attribute_names, header_level=header_level)) if line.startswith("$assertions"): assertions_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}complexType[@name='TestAssertions']") assertion_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}group[@name='TestAssertion']") @@ -132,15 +102,16 @@ def _get_bp_link(annotation_el): anchor = annotation_el.attrib.get("{http://galaxyproject.org/xml/1.0}best_practices", None) link = None if anchor: - link = "http://planemo.readthedocs.io/en/latest/standards/docs/best_practices/tool_xml.html#%s" % anchor + link = "https://planemo.readthedocs.io/en/latest/standards/docs/best_practices/tool_xml.html#%s" % anchor return link -def _build_attributes_table(tag, attributes, hide_attributes=False, attribute_names=None): +def _build_attributes_table(tag, attributes, hide_attributes=False, attribute_names=None, header_level=3): attribute_table = StringIO() attribute_table.write("\n\n") if attributes and not hide_attributes: - attribute_table.write("\n### Attributes\n") + header_prefix = '#' * header_level + attribute_table.write("\n%s Attributes\n" % header_prefix) attribute_table.write("Attribute | Details | Required\n") attribute_table.write("--- | --- | ---\n") for attribute in attributes: diff --git a/doc/patch.py b/doc/patch.py index c6a24111ad9..60a030c3f6a 100755 --- a/doc/patch.py +++ b/doc/patch.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """ Patch utility to apply unified diffs - Brute-force line-by-line non-recursive parsing + Brute-force line-by-line non-recursive parsing Copyright (c) 2008-2012 anatoly techtonik Available under the terms of MIT license @@ -14,22 +14,23 @@ This program needs further tweaking for how we use it at Galaxy. """ +from __future__ import print_function + +import logging +import os +import re +import shutil +import sys +from optparse import OptionParser +from os.path import abspath, exists, isfile + +from six import Iterator, StringIO +from six.moves.urllib.request import urlopen __author__ = "anatoly techtonik " __version__ = "1.12.11" -import copy -import logging -import re -# cStringIO doesn't support unicode in 2.5 -from StringIO import StringIO -import urllib2 - -from os.path import exists, isfile, abspath -import os -import shutil - -#------------------------------------------------ +# ----------------------------------------------- # Logging is controlled by logger named after the # module name (e.g. 'patch' for patch.py module) @@ -41,21 +42,25 @@ debug = logger.debug info = logger.info warning = logger.warning + class NullHandler(logging.Handler): - """ Copied from Python 2.7 to avoid getting - `No handlers could be found for logger "patch"` - http://bugs.python.org/issue16539 - """ - def handle(self, record): - pass - def emit(self, record): - pass - def createLock(self): - self.lock = None + """ Copied from Python 2.7 to avoid getting + `No handlers could be found for logger "patch"` + http://bugs.python.org/issue16539 + """ + + def handle(self, record): + pass + + def emit(self, record): + pass + + def createLock(self): + self.lock = None logger.addHandler(NullHandler()) -#------------------------------------------------ +# ----------------------------------------------- # Constants for Patch/PatchSet types DIFF = PLAIN = "plain" @@ -67,7 +72,7 @@ SVN = SUBVERSION = "svn" MIXED = MIXED = "mixed" -#------------------------------------------------ +# ----------------------------------------------- # Helpers (these could come with Python stdlib) # x...() function are used to work with paths in @@ -75,118 +80,120 @@ MIXED = MIXED = "mixed" # slashes even on Windows. def xisabs(filename): - """ Cross-platform version of `os.path.isabs()` - Returns True if `filename` is absolute on - Linux, OS X or Windows. - """ - if filename.startswith('/'): # Linux/Unix - return True - elif filename.startswith('\\'): # Windows - return True - elif re.match(r'\w:[\\/]', filename): # Windows - return True - return False + """ Cross-platform version of `os.path.isabs()` + Returns True if `filename` is absolute on + Linux, OS X or Windows. + """ + if filename.startswith('/'): # Linux/Unix + return True + elif filename.startswith('\\'): # Windows + return True + elif re.match(r'\w:[\\/]', filename): # Windows + return True + return False + def xnormpath(path): - """ Cross-platform version of os.path.normpath """ - return os.path.normpath(path).replace(os.sep, '/') + """ Cross-platform version of os.path.normpath """ + return os.path.normpath(path).replace(os.sep, '/') + def xstrip(filename): - """ Make relative path out of absolute by stripping - prefixes used on Linux, OS X and Windows. + """ Make relative path out of absolute by stripping + prefixes used on Linux, OS X and Windows. - This function is critical for security. - """ - while xisabs(filename): - # strip windows drive with all slashes - if re.match(r'\w:[\\/]', filename): - filename = re.sub(r'^\w+:[\\/]+', '', filename) - # strip all slashes - elif re.match(r'[\\/]', filename): - filename = re.sub(r'^[\\/]+', '', filename) - return filename + This function is critical for security. + """ + while xisabs(filename): + # strip windows drive with all slashes + if re.match(r'\w:[\\/]', filename): + filename = re.sub(r'^\w+:[\\/]+', '', filename) + # strip all slashes + elif re.match(r'[\\/]', filename): + filename = re.sub(r'^[\\/]+', '', filename) + return filename -#----------------------------------------------- + +# ---------------------------------------------- # Main API functions def fromfile(filename): - """ Parse patch file. If successful, returns - PatchSet() object. Otherwise returns False. - """ - patchset = PatchSet() - debug("reading %s" % filename) - fp = open(filename, "rb") - res = patchset.parse(fp) - fp.close() - if res is True: - return patchset - return False + """ Parse patch file. If successful, returns + PatchSet() object. Otherwise returns False. + """ + patchset = PatchSet() + debug("reading %s" % filename) + fp = open(filename, "rb") + res = patchset.parse(fp) + fp.close() + if res is True: + return patchset + return False def fromstring(s): - """ Parse text string and return PatchSet() - object (or False if parsing fails) - """ - ps = PatchSet( StringIO(s) ) - if ps.errors == 0: - return ps - return False + """ Parse text string and return PatchSet() + object (or False if parsing fails) + """ + ps = PatchSet( StringIO(s) ) + if ps.errors == 0: + return ps + return False def fromurl(url): - """ Parse patch from an URL, return False - if an error occured. Note that this also - can throw urlopen() exceptions. - """ - ps = PatchSet( urllib2.urlopen(url) ) - if ps.errors == 0: - return ps - return False + """ Parse patch from an URL, return False + if an error occured. Note that this also + can throw urlopen() exceptions. + """ + ps = PatchSet( urlopen(url) ) + if ps.errors == 0: + return ps + return False # --- Utility functions --- # [ ] reuse more universal pathsplit() def pathstrip(path, n): - """ Strip n leading components from the given path """ - pathlist = [path] - while os.path.dirname(pathlist[0]) != '': - pathlist[0:1] = os.path.split(pathlist[0]) - return '/'.join(pathlist[n:]) + """ Strip n leading components from the given path """ + pathlist = [path] + while os.path.dirname(pathlist[0]) != '': + pathlist[0:1] = os.path.split(pathlist[0]) + return '/'.join(pathlist[n:]) # --- /Utility function --- class Hunk(object): - """ Parsed hunk data container (hunk starts with @@ -R +R @@) """ + """ Parsed hunk data container (hunk starts with @@ -R +R @@) """ - def __init__(self): - self.startsrc=None #: line count starts with 1 - self.linessrc=None - self.starttgt=None - self.linestgt=None - self.invalid=False - self.hasplus=False # True if any "+" lines in hunk - self.hasminus=False # True if any "-" lines in hunk - self.text=[] + def __init__(self): + self.startsrc = None # line count starts with 1 + self.linessrc = None + self.starttgt = None + self.linestgt = None + self.invalid = False + self.hasplus = False # True if any "+" lines in hunk + self.hasminus = False # True if any "-" lines in hunk + self.text = [] - def originalText(self): + def originalText(self): - return("@@ -" + str(self.startsrc) + - "," + str(self.linessrc) + - " +" + str(self.starttgt) + - "," + str(self.linestgt) + - "\n" + - self.printableText()) + return("@@ -" + str(self.startsrc) + + "," + str(self.linessrc) + + " +" + str(self.starttgt) + + "," + str(self.linestgt) + + "\n" + + self.printableText()) - def printableText(self): - """Reformat text into printable text""" - - # yeah, there must be a better way to do this. - printable = "" - for line in self.text: - printable += line + def printableText(self): + """Reformat text into printable text""" - return printable + # yeah, there must be a better way to do this. + printable = "" + for line in self.text: + printable += line + return printable # def apply(self, estream): @@ -201,885 +208,868 @@ class Hunk(object): class Patch(object): - """ Patch for a single file """ - def __init__(self): - self.source = None - self.target = None - self.hunks = [] - self.hunkends = [] - self.header = [] + """ Patch for a single file """ - self.type = None + def __init__(self): + self.source = None + self.target = None + self.hunks = [] + self.hunkends = [] + self.header = [] + + self.type = None -class PatchSet(object): +class PatchSet(Iterator): - def __init__(self, stream=None): - # --- API accessible fields --- + def __init__(self, stream=None): + # --- API accessible fields --- - # name of the PatchSet (filename or ...) - self.name = None - # patch set type - one of constants - self.type = None + # name of the PatchSet (filename or ...) + self.name = None + # patch set type - one of constants + self.type = None - # list of Patch objects - self.items = [] + # list of Patch objects + self.items = [] - self.errors = 0 # fatal parsing errors - self.warnings = 0 # non-critical warnings - # --- /API --- + self.errors = 0 # fatal parsing errors + self.warnings = 0 # non-critical warnings + # --- /API --- - if stream: - self.parse(stream) + if stream: + self.parse(stream) - def __len__(self): - return len(self.items) + def __len__(self): + return len(self.items) - def parse(self, stream): - """ parse unified diff - return True on success - """ - lineends = dict(lf=0, crlf=0, cr=0) - nexthunkno = 0 #: even if index starts with 0 user messages number hunks from 1 + def parse(self, stream): + """ parse unified diff + return True on success + """ + lineends = dict(lf=0, crlf=0, cr=0) + nexthunkno = 0 #: even if index starts with 0 user messages number hunks from 1 - p = None - hunk = None - # hunkactual variable is used to calculate hunk lines for comparison - hunkactual = dict(linessrc=None, linestgt=None) + p = None + hunk = None + # hunkactual variable is used to calculate hunk lines for comparison + hunkactual = dict(linessrc=None, linestgt=None) + class wrapumerate(enumerate): + """Enumerate wrapper that uses boolean end of stream status instead of + StopIteration exception, and properties to access line information. + """ - class wrapumerate(enumerate): - """Enumerate wrapper that uses boolean end of stream status instead of - StopIteration exception, and properties to access line information. - """ + def __init__(self, *args, **kwargs): + # we don't call parent, it is magically created by __new__ method + self._exhausted = False + self._lineno = False # after end of stream equal to the num of lines + self._line = False # will be reset to False after end of stream - def __init__(self, *args, **kwargs): - # we don't call parent, it is magically created by __new__ method + def __next__(self): + """Try to read the next line and return True if it is available, + False if end of stream is reached.""" + if self._exhausted: + return False - self._exhausted = False - self._lineno = False # after end of stream equal to the num of lines - self._line = False # will be reset to False after end of stream + try: + self._lineno, self._line = next(super(wrapumerate, self)) + except StopIteration: + self._exhausted = True + self._line = False + return False + return True - def next(self): - """Try to read the next line and return True if it is available, - False if end of stream is reached.""" - if self._exhausted: - return False + @property + def is_empty(self): + return self._exhausted - try: - self._lineno, self._line = super(wrapumerate, self).next() - except StopIteration: - self._exhausted = True - self._line = False - return False - return True + @property + def line(self): + return self._line - @property - def is_empty(self): - return self._exhausted + @property + def lineno(self): + return self._lineno - @property - def line(self): - return self._line + # define states (possible file regions) that direct parse flow + headscan = True # start with scanning header + filenames = False # lines starting with --- and +++ - @property - def lineno(self): - return self._lineno + hunkhead = False # @@ -R +R @@ sequence + hunkbody = False # + hunkskip = False # skipping invalid hunk mode - # define states (possible file regions) that direct parse flow - headscan = True # start with scanning header - filenames = False # lines starting with --- and +++ + hunkparsed = False # state after successfully parsed hunk - hunkhead = False # @@ -R +R @@ sequence - hunkbody = False # - hunkskip = False # skipping invalid hunk mode + # regexp to match start of hunk, used groups - 1,3,4,6 + re_hunk_start = re.compile("^@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))?") - hunkparsed = False # state after successfully parsed hunk + self.errors = 0 + # temp buffers for header and filenames info + header = [] + srcname = None + tgtname = None - # regexp to match start of hunk, used groups - 1,3,4,6 - re_hunk_start = re.compile("^@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))?") - - self.errors = 0 - # temp buffers for header and filenames info - header = [] - srcname = None - tgtname = None + # start of main cycle + # each parsing block already has line available in fe.line + fe = wrapumerate(stream) + while next(fe): - # start of main cycle - # each parsing block already has line available in fe.line - fe = wrapumerate(stream) - while fe.next(): + # -- deciders: these only switch state to decide who should process + # -- line fetched at the start of this cycle + if hunkparsed: + hunkparsed = False + if re_hunk_start.match(fe.line): + hunkhead = True + elif fe.line.startswith("--- "): + filenames = True + else: + headscan = True + # -- ------------------------------------ - # -- deciders: these only switch state to decide who should process - # -- line fetched at the start of this cycle - if hunkparsed: - hunkparsed = False - if re_hunk_start.match(fe.line): - hunkhead = True - elif fe.line.startswith("--- "): - filenames = True - else: - headscan = True - # -- ------------------------------------ + # read out header + if headscan: + while not fe.is_empty and not fe.line.startswith("--- "): + header.append(fe.line) + next(fe) + if fe.is_empty: + if p is None: + debug("no patch data found") # error is shown later + self.errors += 1 + else: + info("%d unparsed bytes left at the end of stream" % len(''.join(header))) + self.warnings += 1 + # TODO check for \No new line at the end.. + # TODO test for unparsed bytes + # otherwise error += 1 + # this is actually a loop exit + continue - # read out header - if headscan: - while not fe.is_empty and not fe.line.startswith("--- "): - header.append(fe.line) - fe.next() - if fe.is_empty: - if p == None: - debug("no patch data found") # error is shown later - self.errors += 1 + headscan = False + # switch to filenames state + filenames = True + + line = fe.line + lineno = fe.lineno + + # hunkskip and hunkbody code skipped until definition of hunkhead is parsed + if hunkbody: + # [x] treat empty lines inside hunks as containing single space + # (this happens when diff is saved by copy/pasting to editor + # that strips trailing whitespace) + if line.strip("\r\n") == "": + debug("expanding empty line in a middle of hunk body") + self.warnings += 1 + line = ' ' + line + + # process line first + if re.match(r"^[- \+\\]", line): + # gather stats about line endings + if line.endswith("\r\n"): + p.hunkends["crlf"] += 1 + elif line.endswith("\n"): + p.hunkends["lf"] += 1 + elif line.endswith("\r"): + p.hunkends["cr"] += 1 + + if line.startswith("-"): + hunkactual["linessrc"] += 1 + hunk.hasminus = True + elif line.startswith("+"): + hunkactual["linestgt"] += 1 + hunk.hasplus = True + elif not line.startswith("\\"): + hunkactual["linessrc"] += 1 + hunkactual["linestgt"] += 1 + hunk.text.append(line) + # todo: handle \ No newline cases + else: + warning("invalid hunk no.%d at %d for target file %s" % (nexthunkno, lineno + 1, p.target)) + # add hunk status node + hunk.invalid = True + p.hunks.append(hunk) + self.errors += 1 + # switch to hunkskip state + hunkbody = False + hunkskip = True + + # check exit conditions + if hunkactual["linessrc"] > hunk.linessrc or hunkactual["linestgt"] > hunk.linestgt: + warning("extra lines for hunk no.%d at %d for target %s" % (nexthunkno, lineno + 1, p.target)) + # add hunk status node + hunk.invalid = True + p.hunks.append(hunk) + self.errors += 1 + # switch to hunkskip state + hunkbody = False + hunkskip = True + elif hunk.linessrc == hunkactual["linessrc"] and hunk.linestgt == hunkactual["linestgt"]: + # hunk parsed successfully + p.hunks.append(hunk) + # switch to hunkparsed state + hunkbody = False + hunkparsed = True + + # detect mixed window/unix line ends + ends = p.hunkends + if ((ends["cr"] != 0) + (ends["crlf"] != 0) + (ends["lf"] != 0)) > 1: + warning("inconsistent line ends in patch hunks for %s" % p.source) + self.warnings += 1 + if debugmode: + debuglines = dict(ends) + debuglines.update(file=p.target, hunk=nexthunkno) + debug("crlf: %(crlf)d lf: %(lf)d cr: %(cr)d\t - file: %(file)s hunk: %(hunk)d" % debuglines) + # fetch next line + continue + + if hunkskip: + if re_hunk_start.match(line): + # switch to hunkhead state + hunkskip = False + hunkhead = True + elif line.startswith("--- "): + # switch to filenames state + hunkskip = False + filenames = True + if debugmode and len(self.items) > 0: + debug("- %2d hunks for %s" % (len(p.hunks), p.source)) + + if filenames: + if line.startswith("--- "): + if srcname is not None: + # XXX testcase + warning("skipping false patch for %s" % srcname) + srcname = None + # XXX header += srcname + # double source filename line is encountered + # attempt to restart from this second line + re_filename = "^--- ([^\t]+)" + match = re.match(re_filename, line) + # todo: support spaces in filenames + if match: + srcname = match.group(1).strip() + else: + warning("skipping invalid filename at line %d" % lineno) + self.errors += 1 + # XXX p.header += line + # switch back to headscan state + filenames = False + headscan = True + elif not line.startswith("+++ "): + if srcname is not None: + warning("skipping invalid patch with no target for %s" % srcname) + self.errors += 1 + srcname = None + # XXX header += srcname + # XXX header += line + else: + # this should be unreachable + warning("skipping invalid target patch") + filenames = False + headscan = True + else: + if tgtname is not None: + # XXX seems to be a dead branch + warning("skipping invalid patch - double target at line %d" % lineno) + self.errors += 1 + srcname = None + tgtname = None + # XXX header += srcname + # XXX header += tgtname + # XXX header += line + # double target filename line is encountered + # switch back to headscan state + filenames = False + headscan = True + else: + re_filename = "^\+\+\+ ([^\t]+)" + match = re.match(re_filename, line) + if not match: + warning("skipping invalid patch - no target filename at line %d" % lineno) + self.errors += 1 + srcname = None + # switch back to headscan state + filenames = False + headscan = True + else: + if p: # for the first run p is None + self.items.append(p) + p = Patch() + p.source = srcname + srcname = None + p.target = match.group(1).strip() + p.header = header + header = [] + # switch to hunkhead state + filenames = False + hunkhead = True + nexthunkno = 0 + p.hunkends = lineends.copy() + continue + + if hunkhead: + match = re.match("^@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))?", line) + if not match: + if not p.hunks: + warning("skipping invalid patch with no hunks for file %s" % p.source) + self.errors += 1 + # XXX review switch + # switch to headscan state + hunkhead = False + headscan = True + continue + else: + # TODO review condition case + # switch to headscan state + hunkhead = False + headscan = True + else: + hunk = Hunk() + hunk.startsrc = int(match.group(1)) + hunk.linessrc = 1 + if match.group(3): + hunk.linessrc = int(match.group(3)) + hunk.starttgt = int(match.group(4)) + hunk.linestgt = 1 + if match.group(6): + hunk.linestgt = int(match.group(6)) + hunk.invalid = False + hunk.text = [] + + hunkactual["linessrc"] = hunkactual["linestgt"] = 0 + + # switch to hunkbody state + hunkhead = False + hunkbody = True + nexthunkno += 1 + continue + + # /while fe.next() + + if p: + self.items.append(p) + + if not hunkparsed: + if hunkskip: + warning("warning: finished with errors, some hunks may be invalid") + elif headscan: + if len(self.items) == 0: + warning("error: no patch data found!") + return False + else: # extra data at the end of file + pass else: - info("%d unparsed bytes left at the end of stream" % len(''.join(header))) - self.warnings += 1 - # TODO check for \No new line at the end.. - # TODO test for unparsed bytes - # otherwise error += 1 - # this is actually a loop exit - continue + warning("error: patch stream is incomplete!") + self.errors += 1 + if len(self.items) == 0: + return False - headscan = False - # switch to filenames state - filenames = True - - line = fe.line - lineno = fe.lineno - - - # hunkskip and hunkbody code skipped until definition of hunkhead is parsed - if hunkbody: - # [x] treat empty lines inside hunks as containing single space - # (this happens when diff is saved by copy/pasting to editor - # that strips trailing whitespace) - if line.strip("\r\n") == "": - debug("expanding empty line in a middle of hunk body") - self.warnings += 1 - line = ' ' + line - - # process line first - if re.match(r"^[- \+\\]", line): - # gather stats about line endings - if line.endswith("\r\n"): - p.hunkends["crlf"] += 1 - elif line.endswith("\n"): - p.hunkends["lf"] += 1 - elif line.endswith("\r"): - p.hunkends["cr"] += 1 - - if line.startswith("-"): - hunkactual["linessrc"] += 1 - hunk.hasminus = True - elif line.startswith("+"): - hunkactual["linestgt"] += 1 - hunk.hasplus = True - elif not line.startswith("\\"): - hunkactual["linessrc"] += 1 - hunkactual["linestgt"] += 1 - hunk.text.append(line) - # todo: handle \ No newline cases - else: - warning("invalid hunk no.%d at %d for target file %s" % (nexthunkno, lineno+1, p.target)) - # add hunk status node - hunk.invalid = True - p.hunks.append(hunk) - self.errors += 1 - # switch to hunkskip state - hunkbody = False - hunkskip = True - - # check exit conditions - if hunkactual["linessrc"] > hunk.linessrc or hunkactual["linestgt"] > hunk.linestgt: - warning("extra lines for hunk no.%d at %d for target %s" % (nexthunkno, lineno+1, p.target)) - # add hunk status node - hunk.invalid = True - p.hunks.append(hunk) - self.errors += 1 - # switch to hunkskip state - hunkbody = False - hunkskip = True - elif hunk.linessrc == hunkactual["linessrc"] and hunk.linestgt == hunkactual["linestgt"]: - # hunk parsed successfully - p.hunks.append(hunk) - # switch to hunkparsed state - hunkbody = False - hunkparsed = True - - # detect mixed window/unix line ends - ends = p.hunkends - if ((ends["cr"]!=0) + (ends["crlf"]!=0) + (ends["lf"]!=0)) > 1: - warning("inconsistent line ends in patch hunks for %s" % p.source) - self.warnings += 1 - if debugmode: - debuglines = dict(ends) - debuglines.update(file=p.target, hunk=nexthunkno) - debug("crlf: %(crlf)d lf: %(lf)d cr: %(cr)d\t - file: %(file)s hunk: %(hunk)d" % debuglines) - # fetch next line - continue - - if hunkskip: - if re_hunk_start.match(line): - # switch to hunkhead state - hunkskip = False - hunkhead = True - elif line.startswith("--- "): - # switch to filenames state - hunkskip = False - filenames = True - if debugmode and len(self.items) > 0: + if debugmode and len(self.items) > 0: debug("- %2d hunks for %s" % (len(p.hunks), p.source)) - if filenames: - if line.startswith("--- "): - if srcname != None: - # XXX testcase - warning("skipping false patch for %s" % srcname) - srcname = None - # XXX header += srcname - # double source filename line is encountered - # attempt to restart from this second line - re_filename = "^--- ([^\t]+)" - match = re.match(re_filename, line) - # todo: support spaces in filenames - if match: - srcname = match.group(1).strip() - else: - warning("skipping invalid filename at line %d" % lineno) - self.errors += 1 - # XXX p.header += line - # switch back to headscan state - filenames = False - headscan = True - elif not line.startswith("+++ "): - if srcname != None: - warning("skipping invalid patch with no target for %s" % srcname) - self.errors += 1 - srcname = None - # XXX header += srcname - # XXX header += line - else: - # this should be unreachable - warning("skipping invalid target patch") - filenames = False - headscan = True + # XXX fix total hunks calculation + debug("total files: %d total hunks: %d" % (len(self.items), + sum(len(p.hunks) for p in self.items))) + + # ---- detect patch and patchset types ---- + for idx, p in enumerate(self.items): + self.items[idx].type = self._detect_type(p) + + types = set([p.type for p in self.items]) + if len(types) > 1: + self.type = MIXED else: - if tgtname != None: - # XXX seems to be a dead branch - warning("skipping invalid patch - double target at line %d" % lineno) - self.errors += 1 - srcname = None - tgtname = None - # XXX header += srcname - # XXX header += tgtname - # XXX header += line - # double target filename line is encountered - # switch back to headscan state - filenames = False - headscan = True - else: - re_filename = "^\+\+\+ ([^\t]+)" - match = re.match(re_filename, line) - if not match: - warning("skipping invalid patch - no target filename at line %d" % lineno) - self.errors += 1 - srcname = None - # switch back to headscan state - filenames = False - headscan = True + self.type = types.pop() + # -------- + + self._normalize_filenames() + + return (self.errors == 0) + + def _detect_type(self, p): + """ detect and return type for the specified Patch object + analyzes header and filenames info + + NOTE: must be run before filenames are normalized + """ + + # check for SVN + # - header starts with Index: + # - next line is ===... delimiter + # - filename is followed by revision number + # TODO add SVN revision + if (len(p.header) > 1 and p.header[-2].startswith("Index: ") + and p.header[-1].startswith("=" * 67)): + return SVN + + # common checks for both HG and GIT + DVCS = ((p.source.startswith('a/') or p.source == '/dev/null') + and (p.target.startswith('b/') or p.target == '/dev/null')) + + # GIT type check + # - header[-2] is like "diff --git a/oldname b/newname" + # - header[-1] is like "index .. " + # TODO add git rename diffs and add/remove diffs + # add git diff with spaced filename + # TODO http://www.kernel.org/pub/software/scm/git/docs/git-diff.html + + # detect the start of diff header - there might be some comments before + if len(p.header) > 1: + for idx in reversed(range(len(p.header))): + if p.header[idx].startswith("diff --git"): + break + if re.match(r'diff --git a/[\w/.]+ b/[\w/.]+', p.header[idx]): + if (idx + 1 < len(p.header) + and re.match(r'index \w{7}..\w{7} \d{6}', p.header[idx + 1])): + if DVCS: + return GIT + + # HG check + # + # - for plain HG format header is like "diff -r b2d9961ff1f5 filename" + # - for Git-style HG patches it is "diff --git a/oldname b/newname" + # - filename starts with a/, b/ or is equal to /dev/null + # - exported changesets also contain the header + # # HG changeset patch + # # User name@example.com + # ... + # TODO add MQ + # TODO add revision info + if len(p.header) > 0: + if DVCS and re.match(r'diff -r \w{12} .*', p.header[-1]): + return HG + if DVCS and p.header[-1].startswith('diff --git a/'): + if len(p.header) == 1: # native Git patch header len is 2 + return HG + elif p.header[0].startswith('# HG changeset patch'): + return HG + + return PLAIN + + def _normalize_filenames(self): + """ sanitize filenames, normalizing paths, i.e.: + 1. strip a/ and b/ prefixes from GIT and HG style patches + 2. remove all references to parent directories (with warning) + 3. translate any absolute paths to relative (with warning) + + [x] always use forward slashes to be crossplatform + (diff/patch were born as a unix utility after all) + + return None + """ + for i, p in enumerate(self.items): + if p.type in (HG, GIT): + # TODO: figure out how to deal with /dev/null entries + debug("stripping a/ and b/ prefixes") + if p.source != '/dev/null': + if not p.source.startswith("a/"): + warning("invalid source filename") + else: + p.source = p.source[2:] + if p.target != '/dev/null': + if not p.target.startswith("b/"): + warning("invalid target filename") + else: + p.target = p.target[2:] + + p.source = xnormpath(p.source) + p.target = xnormpath(p.target) + + sep = '/' # sep value can be hardcoded, but it looks nice this way + + # references to parent are not allowed + if p.source.startswith(".." + sep): + warning("error: stripping parent path for source file patch no.%d" % (i + 1)) + self.warnings += 1 + while p.source.startswith(".." + sep): + p.source = p.source.partition(sep)[2] + if p.target.startswith(".." + sep): + warning("error: stripping parent path for target file patch no.%d" % (i + 1)) + self.warnings += 1 + while p.target.startswith(".." + sep): + p.target = p.target.partition(sep)[2] + # absolute paths are not allowed + if xisabs(p.source) or xisabs(p.target): + warning("error: absolute paths are not allowed - file no.%d" % (i + 1)) + self.warnings += 1 + if xisabs(p.source): + warning("stripping absolute path from source name '%s'" % p.source) + p.source = xstrip(p.source) + if xisabs(p.target): + warning("stripping absolute path from target name '%s'" % p.target) + p.target = xstrip(p.target) + + self.items[i].source = p.source + self.items[i].target = p.target + + def diffstat(self): + """ calculate diffstat and return as a string + Notes: + - original diffstat ouputs target filename + - single + or - shouldn't escape histogram + """ + names = [] + insert = [] + delete = [] + namelen = 0 + # max number of changes for single file (for histogram width + # calculation) + maxdiff = 0 + for patch in self.items: + i, d = 0, 0 + for hunk in patch.hunks: + for line in hunk.text: + if line.startswith('+'): + i += 1 + elif line.startswith('-'): + d += 1 + names.append(patch.target) + insert.append(i) + delete.append(d) + namelen = max(namelen, len(patch.target)) + maxdiff = max(maxdiff, i + d) + output = '' + statlen = len(str(maxdiff)) # stats column width + for i, n in enumerate(names): + # %-19s | %-4d %s + format = " %-" + str(namelen) + "s | %" + str(statlen) + "s %s\n" + + hist = '' + # -- calculating histogram -- + width = len(format % ('', '', '')) + histwidth = max(2, 80 - width) + if maxdiff < histwidth: + hist = "+" * insert[i] + "-" * delete[i] else: - if p: # for the first run p is None - self.items.append(p) - p = Patch() - p.source = srcname - srcname = None - p.target = match.group(1).strip() - p.header = header - header = [] - # switch to hunkhead state - filenames = False - hunkhead = True - nexthunkno = 0 - p.hunkends = lineends.copy() - continue + iratio = (float(insert[i]) / maxdiff) * histwidth + dratio = (float(delete[i]) / maxdiff) * histwidth - if hunkhead: - match = re.match("^@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))?", line) - if not match: - if not p.hunks: - warning("skipping invalid patch with no hunks for file %s" % p.source) - self.errors += 1 - # XXX review switch - # switch to headscan state - hunkhead = False - headscan = True - continue - else: - # TODO review condition case - # switch to headscan state - hunkhead = False - headscan = True - else: - hunk = Hunk() - hunk.startsrc = int(match.group(1)) - hunk.linessrc = 1 - if match.group(3): hunk.linessrc = int(match.group(3)) - hunk.starttgt = int(match.group(4)) - hunk.linestgt = 1 - if match.group(6): hunk.linestgt = int(match.group(6)) - hunk.invalid = False - hunk.text = [] + # make sure every entry gets at least one + or - + iwidth = 1 if 0 < iratio < 1 else int(iratio) + dwidth = 1 if 0 < dratio < 1 else int(dratio) + hist = "+" * int(iwidth) + "-" * int(dwidth) + # -- /calculating +- histogram -- + output += (format % (names[i], insert[i] + delete[i], hist)) - hunkactual["linessrc"] = hunkactual["linestgt"] = 0 + output += (" %d files changed, %d insertions(+), %d deletions(-)" + % (len(names), sum(insert), sum(delete))) + return output - # switch to hunkbody state - hunkhead = False - hunkbody = True - nexthunkno += 1 - continue + def apply(self, strip=0): + """ apply parsed patch + return True on success + """ - # /while fe.next() - - if p: - self.items.append(p) - - if not hunkparsed: - if hunkskip: - warning("warning: finished with errors, some hunks may be invalid") - elif headscan: - if len(self.items) == 0: - warning("error: no patch data found!") - return False - else: # extra data at the end of file - pass - else: - warning("error: patch stream is incomplete!") - self.errors += 1 - if len(self.items) == 0: - return False - - if debugmode and len(self.items) > 0: - debug("- %2d hunks for %s" % (len(p.hunks), p.source)) - - # XXX fix total hunks calculation - debug("total files: %d total hunks: %d" % (len(self.items), - sum(len(p.hunks) for p in self.items))) - - # ---- detect patch and patchset types ---- - for idx, p in enumerate(self.items): - self.items[idx].type = self._detect_type(p) - - types = set([p.type for p in self.items]) - if len(types) > 1: - self.type = MIXED - else: - self.type = types.pop() - # -------- - - self._normalize_filenames() - - return (self.errors == 0) - - def _detect_type(self, p): - """ detect and return type for the specified Patch object - analyzes header and filenames info - - NOTE: must be run before filenames are normalized - """ - - # check for SVN - # - header starts with Index: - # - next line is ===... delimiter - # - filename is followed by revision number - # TODO add SVN revision - if (len(p.header) > 1 and p.header[-2].startswith("Index: ") - and p.header[-1].startswith("="*67)): - return SVN - - # common checks for both HG and GIT - DVCS = ((p.source.startswith('a/') or p.source == '/dev/null') - and (p.target.startswith('b/') or p.target == '/dev/null')) - - # GIT type check - # - header[-2] is like "diff --git a/oldname b/newname" - # - header[-1] is like "index .. " - # TODO add git rename diffs and add/remove diffs - # add git diff with spaced filename - # TODO http://www.kernel.org/pub/software/scm/git/docs/git-diff.html - - # detect the start of diff header - there might be some comments before - if len(p.header) > 1: - for idx in reversed(range(len(p.header))): - if p.header[idx].startswith("diff --git"): - break - if re.match(r'diff --git a/[\w/.]+ b/[\w/.]+', p.header[idx]): - if (idx+1 < len(p.header) - and re.match(r'index \w{7}..\w{7} \d{6}', p.header[idx+1])): - if DVCS: - return GIT - - # HG check - # - # - for plain HG format header is like "diff -r b2d9961ff1f5 filename" - # - for Git-style HG patches it is "diff --git a/oldname b/newname" - # - filename starts with a/, b/ or is equal to /dev/null - # - exported changesets also contain the header - # # HG changeset patch - # # User name@example.com - # ... - # TODO add MQ - # TODO add revision info - if len(p.header) > 0: - if DVCS and re.match(r'diff -r \w{12} .*', p.header[-1]): - return HG - if DVCS and p.header[-1].startswith('diff --git a/'): - if len(p.header) == 1: # native Git patch header len is 2 - return HG - elif p.header[0].startswith('# HG changeset patch'): - return HG - - return PLAIN - - - def _normalize_filenames(self): - """ sanitize filenames, normalizing paths, i.e.: - 1. strip a/ and b/ prefixes from GIT and HG style patches - 2. remove all references to parent directories (with warning) - 3. translate any absolute paths to relative (with warning) - - [x] always use forward slashes to be crossplatform - (diff/patch were born as a unix utility after all) - - return None - """ - for i,p in enumerate(self.items): - if p.type in (HG, GIT): - # TODO: figure out how to deal with /dev/null entries - debug("stripping a/ and b/ prefixes") - if p.source != '/dev/null': - if not p.source.startswith("a/"): - warning("invalid source filename") - else: - p.source = p.source[2:] - if p.target != '/dev/null': - if not p.target.startswith("b/"): - warning("invalid target filename") - else: - p.target = p.target[2:] - - p.source = xnormpath(p.source) - p.target = xnormpath(p.target) - - sep = '/' # sep value can be hardcoded, but it looks nice this way - - # references to parent are not allowed - if p.source.startswith(".." + sep): - warning("error: stripping parent path for source file patch no.%d" % (i+1)) - self.warnings += 1 - while p.source.startswith(".." + sep): - p.source = p.source.partition(sep)[2] - if p.target.startswith(".." + sep): - warning("error: stripping parent path for target file patch no.%d" % (i+1)) - self.warnings += 1 - while p.target.startswith(".." + sep): - p.target = p.target.partition(sep)[2] - # absolute paths are not allowed - if xisabs(p.source) or xisabs(p.target): - warning("error: absolute paths are not allowed - file no.%d" % (i+1)) - self.warnings += 1 - if xisabs(p.source): - warning("stripping absolute path from source name '%s'" % p.source) - p.source = xstrip(p.source) - if xisabs(p.target): - warning("stripping absolute path from target name '%s'" % p.target) - p.target = xstrip(p.target) - - self.items[i].source = p.source - self.items[i].target = p.target - - - def diffstat(self): - """ calculate diffstat and return as a string - Notes: - - original diffstat ouputs target filename - - single + or - shouldn't escape histogram - """ - names = [] - insert = [] - delete = [] - namelen = 0 - maxdiff = 0 # max number of changes for single file - # (for histogram width calculation) - for patch in self.items: - i,d = 0,0 - for hunk in patch.hunks: - for line in hunk.text: - if line.startswith('+'): - i += 1 - elif line.startswith('-'): - d += 1 - names.append(patch.target) - insert.append(i) - delete.append(d) - namelen = max(namelen, len(patch.target)) - maxdiff = max(maxdiff, i+d) - output = '' - statlen = len(str(maxdiff)) # stats column width - for i,n in enumerate(names): - # %-19s | %-4d %s - format = " %-" + str(namelen) + "s | %" + str(statlen) + "s %s\n" - - hist = '' - # -- calculating histogram -- - width = len(format % ('', '', '')) - histwidth = max(2, 80 - width) - if maxdiff < histwidth: - hist = "+"*insert[i] + "-"*delete[i] - else: - iratio = (float(insert[i]) / maxdiff) * histwidth - dratio = (float(delete[i]) / maxdiff) * histwidth - - # make sure every entry gets at least one + or - - iwidth = 1 if 0 < iratio < 1 else int(iratio) - dwidth = 1 if 0 < dratio < 1 else int(dratio) - #print iratio, dratio, iwidth, dwidth, histwidth - hist = "+"*int(iwidth) + "-"*int(dwidth) - # -- /calculating +- histogram -- - output += (format % (names[i], insert[i] + delete[i], hist)) - - output += (" %d files changed, %d insertions(+), %d deletions(-)" - % (len(names), sum(insert), sum(delete))) - return output - - - def apply(self, strip=0): - """ apply parsed patch - return True on success - """ - - total = len(self.items) - errors = 0 - if strip: - # [ ] test strip level exceeds nesting level - # [ ] test the same only for selected files - # [ ] test if files end up being on the same level - try: - strip = int(strip) - except ValueError: - errors += 1 - warning("error: strip parameter '%s' must be an integer" % strip) - strip = 0 - - #for fileno, filename in enumerate(self.source): - for i,p in enumerate(self.items): - f2patch = p.source - if strip: - debug("stripping %s leading component from '%s'" % (strip, f2patch)) - f2patch = pathstrip(f2patch, strip) - if not exists(f2patch): - f2patch = p.target + total = len(self.items) + errors = 0 if strip: - debug("stripping %s leading component from '%s'" % (strip, f2patch)) - f2patch = pathstrip(f2patch, strip) - if not exists(f2patch): - warning("source/target file does not exist\n--- %s\n+++ %s" % (p.source, f2patch)) - errors += 1 - continue - if not isfile(f2patch): - warning("not a file - %s" % f2patch) - errors += 1 - continue - filename = f2patch + # [ ] test strip level exceeds nesting level + # [ ] test the same only for selected files + # [ ] test if files end up being on the same level + try: + strip = int(strip) + except ValueError: + errors += 1 + warning("error: strip parameter '%s' must be an integer" % strip) + strip = 0 - debug("processing %d/%d:\t %s" % (i+1, total, filename)) + for i, p in enumerate(self.items): + f2patch = p.source + if strip: + debug("stripping %s leading component from '%s'" % (strip, f2patch)) + f2patch = pathstrip(f2patch, strip) + if not exists(f2patch): + f2patch = p.target + if strip: + debug("stripping %s leading component from '%s'" % (strip, f2patch)) + f2patch = pathstrip(f2patch, strip) + if not exists(f2patch): + warning("source/target file does not exist\n--- %s\n+++ %s" % (p.source, f2patch)) + errors += 1 + continue + if not isfile(f2patch): + warning("not a file - %s" % f2patch) + errors += 1 + continue + filename = f2patch - # validate before patching - f2fp = open(filename) - hunkno = 0 - hunk = p.hunks[hunkno] - hunkfind = [] - hunkreplace = [] - validhunks = 0 - canpatch = False - for lineno, line in enumerate(f2fp): - if lineno+1 < hunk.startsrc: - continue - elif lineno+1 == hunk.startsrc: - hunkfind = [x[1:].rstrip("\r\n") for x in hunk.text if x[0] in " -"] - hunkreplace = [x[1:].rstrip("\r\n") for x in hunk.text if x[0] in " +"] - #pprint(hunkreplace) - hunklineno = 0 + debug("processing %d/%d:\t %s" % (i + 1, total, filename)) - # todo \ No newline at end of file - - # check hunks in source file - if lineno+1 < hunk.startsrc+len(hunkfind)-1: - if line.rstrip("\r\n") == hunkfind[hunklineno]: - hunklineno+=1 - else: - info("file %d/%d:\t %s" % (i+1, total, filename)) - info(" hunk no.%d doesn't match source file at line %d" % (hunkno+1, lineno)) - info(" expected: %s" % hunkfind[hunklineno]) - info(" actual : %s" % line.rstrip("\r\n")) - # not counting this as error, because file may already be patched. - # check if file is already patched is done after the number of - # invalid hunks if found - # TODO: check hunks against source/target file in one pass - # API - check(stream, srchunks, tgthunks) - # return tuple (srcerrs, tgterrs) - - # continue to check other hunks for completeness - hunkno += 1 - if hunkno < len(p.hunks): - hunk = p.hunks[hunkno] - continue - else: - break - - # check if processed line is the last line - if lineno+1 == hunk.startsrc+len(hunkfind)-1: - debug(" hunk no.%d for file %s -- is ready to be patched" % (hunkno+1, filename)) - hunkno+=1 - validhunks+=1 - if hunkno < len(p.hunks): + # validate before patching + f2fp = open(filename) + hunkno = 0 hunk = p.hunks[hunkno] - else: - if validhunks == len(p.hunks): - # patch file - canpatch = True - break - else: - if hunkno < len(p.hunks): - warning("premature end of source file %s at hunk %d" % (filename, hunkno+1)) - errors += 1 + hunkfind = [] + validhunks = 0 + canpatch = False + for lineno, line in enumerate(f2fp): + if lineno + 1 < hunk.startsrc: + continue + elif lineno + 1 == hunk.startsrc: + hunkfind = [x[1:].rstrip("\r\n") for x in hunk.text if x[0] in " -"] + hunklineno = 0 - f2fp.close() + # todo \ No newline at end of file - if validhunks < len(p.hunks): - if self._match_file_hunks(filename, p.hunks): - warning("already patched %s" % filename) - else: - warning("source file is different - %s" % filename) - errors += 1 - if canpatch: - backupname = filename+".orig" - if exists(backupname): - warning("can't backup original file to %s - aborting" % backupname) - else: - import shutil - shutil.move(filename, backupname) - if self.write_hunks(backupname, filename, p.hunks): - info("successfully patched %d/%d:\t %s" % (i+1, total, filename)) - os.unlink(backupname) - else: - errors += 1 - warning("error patching file %s" % filename) - shutil.copy(filename, filename+".invalid") - warning("invalid version is saved to %s" % filename+".invalid") - # todo: proper rejects - shutil.move(backupname, filename) + # check hunks in source file + if lineno + 1 < hunk.startsrc + len(hunkfind) - 1: + if line.rstrip("\r\n") == hunkfind[hunklineno]: + hunklineno += 1 + else: + info("file %d/%d:\t %s" % (i + 1, total, filename)) + info(" hunk no.%d doesn't match source file at line %d" % (hunkno + 1, lineno)) + info(" expected: %s" % hunkfind[hunklineno]) + info(" actual : %s" % line.rstrip("\r\n")) + # not counting this as error, because file may already be patched. + # check if file is already patched is done after the number of + # invalid hunks if found + # TODO: check hunks against source/target file in one pass + # API - check(stream, srchunks, tgthunks) + # return tuple (srcerrs, tgterrs) - # todo: check for premature eof - return (errors == 0) + # continue to check other hunks for completeness + hunkno += 1 + if hunkno < len(p.hunks): + hunk = p.hunks[hunkno] + continue + else: + break + # check if processed line is the last line + if lineno + 1 == hunk.startsrc + len(hunkfind) - 1: + debug(" hunk no.%d for file %s -- is ready to be patched" % (hunkno + 1, filename)) + hunkno += 1 + validhunks += 1 + if hunkno < len(p.hunks): + hunk = p.hunks[hunkno] + else: + if validhunks == len(p.hunks): + # patch file + canpatch = True + break + else: + if hunkno < len(p.hunks): + warning("premature end of source file %s at hunk %d" % (filename, hunkno + 1)) + errors += 1 - def can_patch(self, filename): - """ Check if specified filename can be patched. Returns None if file can - not be found among source filenames. False if patch can not be applied - clearly. True otherwise. + f2fp.close() - :returns: True, False or None - """ - filename = abspath(filename) - for p in self.items: - if filename == abspath(p.source): - return self._match_file_hunks(filename, p.hunks) - return None + if validhunks < len(p.hunks): + if self._match_file_hunks(filename, p.hunks): + warning("already patched %s" % filename) + else: + warning("source file is different - %s" % filename) + errors += 1 + if canpatch: + backupname = filename + ".orig" + if exists(backupname): + warning("can't backup original file to %s - aborting" % backupname) + else: + import shutil + shutil.move(filename, backupname) + if self.write_hunks(backupname, filename, p.hunks): + info("successfully patched %d/%d:\t %s" % (i + 1, total, filename)) + os.unlink(backupname) + else: + errors += 1 + warning("error patching file %s" % filename) + shutil.copy(filename, filename + ".invalid") + warning("invalid version is saved to %s" % filename + ".invalid") + # todo: proper rejects + shutil.move(backupname, filename) + # todo: check for premature eof + return (errors == 0) - def _match_file_hunks(self, filepath, hunks): - matched = True - fp = open(abspath(filepath)) + def can_patch(self, filename): + """ Check if specified filename can be patched. Returns None if file can + not be found among source filenames. False if patch can not be applied + clearly. True otherwise. - class NoMatch(Exception): - pass + :returns: True, False or None + """ + filename = abspath(filename) + for p in self.items: + if filename == abspath(p.source): + return self._match_file_hunks(filename, p.hunks) + return None - lineno = 1 - line = fp.readline() - hno = None - try: - for hno, h in enumerate(hunks): - # skip to first line of the hunk - while lineno < h.starttgt: - if not len(line): # eof - debug("check failed - premature eof before hunk: %d" % (hno+1)) - raise NoMatch - line = fp.readline() - lineno += 1 - for hline in h.text: - if hline.startswith("-"): - continue - if not len(line): - debug("check failed - premature eof on hunk: %d" % (hno+1)) - # todo: \ No newline at the end of file - raise NoMatch - if line.rstrip("\r\n") != hline[1:].rstrip("\r\n"): - debug("file is not patched - failed hunk: %d" % (hno+1)) - raise NoMatch - line = fp.readline() - lineno += 1 + def _match_file_hunks(self, filepath, hunks): + matched = True + fp = open(abspath(filepath)) - except NoMatch: - matched = False - # todo: display failed hunk, i.e. expected/found + class NoMatch(Exception): + pass - fp.close() - return matched + lineno = 1 + line = fp.readline() + hno = None + try: + for hno, h in enumerate(hunks): + # skip to first line of the hunk + while lineno < h.starttgt: + if not len(line): # eof + debug("check failed - premature eof before hunk: %d" % (hno + 1)) + raise NoMatch + line = fp.readline() + lineno += 1 + for hline in h.text: + if hline.startswith("-"): + continue + if not len(line): + debug("check failed - premature eof on hunk: %d" % (hno + 1)) + # todo: \ No newline at the end of file + raise NoMatch + if line.rstrip("\r\n") != hline[1:].rstrip("\r\n"): + debug("file is not patched - failed hunk: %d" % (hno + 1)) + raise NoMatch + line = fp.readline() + lineno += 1 + except NoMatch: + matched = False + # todo: display failed hunk, i.e. expected/found - def patch_stream(self, instream, hunks): - """ Generator that yields stream patched with hunks iterable - - Converts lineends in hunk lines to the best suitable format - autodetected from input - """ + fp.close() + return matched - # todo: At the moment substituted lineends may not be the same - # at the start and at the end of patching. Also issue a - # warning/throw about mixed lineends (is it really needed?) + def patch_stream(self, instream, hunks): + """ Generator that yields stream patched with hunks iterable - hunks = iter(hunks) + Converts lineends in hunk lines to the best suitable format + autodetected from input + """ - srclineno = 1 + # todo: At the moment substituted lineends may not be the same + # at the start and at the end of patching. Also issue a + # warning/throw about mixed lineends (is it really needed?) - lineends = {'\n':0, '\r\n':0, '\r':0} - def get_line(): - """ - local utility function - return line from source stream - collecting line end statistics on the way - """ - line = instream.readline() - # 'U' mode works only with text files - if line.endswith("\r\n"): - lineends["\r\n"] += 1 - elif line.endswith("\n"): - lineends["\n"] += 1 - elif line.endswith("\r"): - lineends["\r"] += 1 - return line + hunks = iter(hunks) - for hno, h in enumerate(hunks): - debug("hunk %d" % (hno+1)) - if h.hasminus: - warning("Change removes/replaces some text; INVESTIGATE AND APPLY (OR NOT) MANUALLY") - warning("Change:") - changeText = h.originalText() - if len(changeText) > 1000: - changeText = changeText[0:999] + "...\n" - warning(changeText) - else: - # skip to line just before hunk starts - while srclineno < h.startsrc: - yield get_line() - srclineno += 1 + srclineno = 1 - for hline in h.text: - # todo: check \ No newline at the end of file - if hline.startswith("-") or hline.startswith("\\"): - get_line() - srclineno += 1 - continue - else: - if not hline.startswith("+"): - get_line() - srclineno += 1 - line2write = hline[1:] - # detect if line ends are consistent in source file - if sum([bool(lineends[x]) for x in lineends]) == 1: - newline = [x for x in lineends if lineends[x] != 0][0] - yield line2write.rstrip("\r\n")+newline - else: # newlines are mixed - yield line2write - - for line in instream: - yield line + lineends = {'\n': 0, '\r\n': 0, '\r': 0} + def get_line(): + """ + local utility function - return line from source stream + collecting line end statistics on the way + """ + line = instream.readline() + # 'U' mode works only with text files + if line.endswith("\r\n"): + lineends["\r\n"] += 1 + elif line.endswith("\n"): + lineends["\n"] += 1 + elif line.endswith("\r"): + lineends["\r"] += 1 + return line - def write_hunks(self, srcname, tgtname, hunks): - src = open(srcname, "rb") - tgt = open(tgtname, "wb") + for hno, h in enumerate(hunks): + debug("hunk %d" % (hno + 1)) + if h.hasminus: + warning("Change removes/replaces some text; INVESTIGATE AND APPLY (OR NOT) MANUALLY") + warning("Change:") + changeText = h.originalText() + if len(changeText) > 1000: + changeText = changeText[0:999] + "...\n" + warning(changeText) + else: + # skip to line just before hunk starts + while srclineno < h.startsrc: + yield get_line() + srclineno += 1 - debug("processing target file %s" % tgtname) + for hline in h.text: + # todo: check \ No newline at the end of file + if hline.startswith("-") or hline.startswith("\\"): + get_line() + srclineno += 1 + continue + else: + if not hline.startswith("+"): + get_line() + srclineno += 1 + line2write = hline[1:] + # detect if line ends are consistent in source file + if sum([bool(lineends[x]) for x in lineends]) == 1: + newline = [x for x in lineends if lineends[x] != 0][0] + yield line2write.rstrip("\r\n") + newline + else: # newlines are mixed + yield line2write - tgt.writelines(self.patch_stream(src, hunks)) + for line in instream: + yield line - tgt.close() - src.close() - # [ ] TODO: add test for permission copy - shutil.copymode(srcname, tgtname) - return True + def write_hunks(self, srcname, tgtname, hunks): + src = open(srcname, "rb") + tgt = open(tgtname, "wb") + debug("processing target file %s" % tgtname) + + tgt.writelines(self.patch_stream(src, hunks)) + + tgt.close() + src.close() + # [ ] TODO: add test for permission copy + shutil.copymode(srcname, tgtname) + return True if __name__ == "__main__": - from optparse import OptionParser - from os.path import exists - import sys + opt = OptionParser(usage="1. %prog [options] unified.diff\n" + " 2. %prog [options] http://host/patch\n" + " 3. %prog [options] -- < unified.diff", + version="python-patch %s" % __version__) + opt.add_option("-q", "--quiet", action="store_const", dest="verbosity", + const=0, help="print only warnings and errors", default=1) + opt.add_option("-v", "--verbose", action="store_const", dest="verbosity", + const=2, help="be verbose") + opt.add_option("--debug", action="store_true", dest="debugmode", help="debug mode") + opt.add_option("--diffstat", action="store_true", dest="diffstat", + help="print diffstat and exit") + opt.add_option("-p", "--strip", type="int", metavar='N', default=0, + help="strip N path components from filenames") + (options, args) = opt.parse_args() - opt = OptionParser(usage="1. %prog [options] unified.diff\n" - " 2. %prog [options] http://host/patch\n" - " 3. %prog [options] -- < unified.diff", - version="python-patch %s" % __version__) - opt.add_option("-q", "--quiet", action="store_const", dest="verbosity", - const=0, help="print only warnings and errors", default=1) - opt.add_option("-v", "--verbose", action="store_const", dest="verbosity", - const=2, help="be verbose") - opt.add_option("--debug", action="store_true", dest="debugmode", help="debug mode") - opt.add_option("--diffstat", action="store_true", dest="diffstat", - help="print diffstat and exit") - opt.add_option("-p", "--strip", type="int", metavar='N', default=0, - help="strip N path components from filenames") - (options, args) = opt.parse_args() + if not args and sys.argv[-1:] != ['--']: + opt.print_version() + opt.print_help() + sys.exit() + readstdin = (sys.argv[-1:] == ['--'] and not args) - if not args and sys.argv[-1:] != ['--']: - opt.print_version() - opt.print_help() - sys.exit() - readstdin = (sys.argv[-1:] == ['--'] and not args) + debugmode = options.debugmode - debugmode = options.debugmode + verbosity_levels = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} + loglevel = verbosity_levels[options.verbosity] + logformat = "%(message)s" + if debugmode: + loglevel = logging.DEBUG + logformat = "%(levelname)8s %(message)s" + logger.setLevel(loglevel) + loghandler = logging.StreamHandler() + loghandler.setFormatter(logging.Formatter(logformat)) + logger.addHandler(loghandler) - verbosity_levels = {0:logging.WARNING, 1:logging.INFO, 2:logging.DEBUG} - loglevel = verbosity_levels[options.verbosity] - logformat = "%(message)s" - if debugmode: - loglevel = logging.DEBUG - logformat = "%(levelname)8s %(message)s" - logger.setLevel(loglevel) - loghandler = logging.StreamHandler() - loghandler.setFormatter(logging.Formatter(logformat)) - logger.addHandler(loghandler) - - - if readstdin: - patch = PatchSet(sys.stdin) - else: - patchfile = args[0] - urltest = patchfile.split(':')[0] - if (':' in patchfile and urltest.isalpha() - and len(urltest) > 1): # one char before : is a windows drive letter - patch = fromurl(patchfile) + if readstdin: + patch = PatchSet(sys.stdin) else: - if not exists(patchfile) or not isfile(patchfile): - sys.exit("patch file does not exist - %s" % patchfile) - patch = fromfile(patchfile) + patchfile = args[0] + urltest = patchfile.split(':')[0] + if (':' in patchfile and urltest.isalpha() + and len(urltest) > 1): # one char before : is a windows drive letter + patch = fromurl(patchfile) + else: + if not exists(patchfile) or not isfile(patchfile): + sys.exit("patch file does not exist - %s" % patchfile) + patch = fromfile(patchfile) - if options.diffstat: - print patch.diffstat() - sys.exit(0) + if options.diffstat: + print(patch.diffstat()) + sys.exit(0) - #pprint(patch) - patch.apply(options.strip) or sys.exit(-1) + patch.apply(options.strip) or sys.exit(-1) - # todo: document and test line ends handling logic - patch.py detects proper line-endings - # for inserted hunks and issues a warning if patched file has incosistent line ends + # todo: document and test line ends handling logic - patch.py detects proper line-endings + # for inserted hunks and issues a warning if patched file has incosistent line ends diff --git a/doc/schema_template.md b/doc/schema_template.md index 239a5cf318b..77d077db7f3 100644 --- a/doc/schema_template.md +++ b/doc/schema_template.md @@ -10,11 +10,9 @@ If you find a bug please report it [here](https://github.com/galaxyproject/galax This document serves as reference documentation. If you would like to learn how to build tools for Galaxy, -[Planemo](http://planemo.readthedocs.io/en/latest/writing.html) features a +[Planemo](https://planemo.readthedocs.io/en/latest/writing.html) features a number of tutorials on building Galaxy tools that would better serve that purpose. -$toc - $tag:tool://element[@name='tool'] $tag:tool|description://element[@name='tool']//element[@name='description'] $tag:tool|version_command://complexType[@name='VersionCommand'] diff --git a/doc/source/admin/conda_faq.rst b/doc/source/admin/conda_faq.rst index 5e51b10dce4..4fe67d155b6 100644 --- a/doc/source/admin/conda_faq.rst +++ b/doc/source/admin/conda_faq.rst @@ -1,6 +1,6 @@ -================================= +=========================== Conda for Tool Dependencies -================================= +=========================== Galaxy tools (also called wrappers) traditionally use Tool Shed package recipes to install their dependencies. At the tool's installation time @@ -46,7 +46,7 @@ Below we answer some common questions (collected by Lance Parsons): 1. How do I enable Conda dependency resolution for Galaxy jobs? -********************************************************************************* +*************************************************************** Galaxy's dependency job resolution is managed via ``dependency_resolvers_conf.xml`` configuration file. Most Galaxy administrators @@ -103,7 +103,7 @@ See `galaxy.ini.sample`_ for the complete list. 2. How do Conda dependencies work? Where do things get installed? -********************************************************************************* +***************************************************************** In contrast to the TS dependency system, which was used exclusively by Galaxy, Conda is a pre-existing, independent project. With Conda, it is possible for an @@ -154,7 +154,7 @@ be configured in the ``dependency_resolvers_conf.xml`` and the ``galaxy.ini`` fi 3. What is required to make use of this? Any specific packages, Galaxy revision, OS version, etc.? -********************************************************************************* +************************************************************************************************** The minimum required version of Galaxy to use Conda is 16.01, however version 16.07 or greater is recommended. The 16.07 release of Galaxy has @@ -169,7 +169,7 @@ systems newer than 2007. 4. If I have Conda enabled, what do I need to do to install tools using it? For example, how can I install the latest Trinity? And how will I know the dependencies are installed? -********************************************************************************* +********************************************************************************************************************************************************************************** This depends on your ``galaxy.ini`` setting. Starting with release 16.07, Galaxy can automatically install the Conda package manager for you if you have enabled @@ -198,7 +198,7 @@ are no available TS dependencies. 5. Can I mix traditional Galaxy packages and Conda packages? -********************************************************************************* +************************************************************ Yes, the way this works is that Galaxy goes through the list of requirements for a tool, and then determines for each requirement if it @@ -216,7 +216,7 @@ The first system that satisfies a requirement will be used. See 6. How do I know what system is being used by a given tool? -********************************************************************************* +*********************************************************** The Galaxy log will show which dependency resolution system is used to satisfy each tool dependency and you can specify priorities using the @@ -226,7 +226,7 @@ Admin panel. 7. How do I go about specifying Conda dependencies for a tool? All the docs still seem to recommend (or exclusively discuss) the ``tool_dependencies.xml`` method. -********************************************************************************* +****************************************************************************************************************************************************************** The simple answer is: you don't need to do much to make Conda work for a tool. @@ -243,7 +243,7 @@ deprecate it everywhere. 8. During tool installation what if there is no Conda package available for a given requirement? What if the requirement is resolved in a different software than the original wrapper author meant to use? -********************************************************************************* +*********************************************************************************************************************************************************************************************************** If there is no Conda package available during tool installation the tool will install automatically, and can be used if its dependencies are @@ -256,7 +256,7 @@ installed from the Tool Shed. 9. Where can I find a list of existing Conda packages that I can point to, so I don't have to reinvent the wheel for common dependencies? -********************************************************************************* +***************************************************************************************************************************************** With Conda package manager installed on your system, run: @@ -270,7 +270,7 @@ Galaxy. If you find your package, you are ready to go. If not please 10. How can I create a new Conda package for a dependency? -********************************************************************************* +********************************************************** Adding a package to the BioConda or IUC Conda channels will make it available for Galaxy tools to use as a dependency. To learn how, get in @@ -284,7 +284,7 @@ pypi, cran, or cpan for you (mostly) automatically. 11. Is there a way to convert traditional Tool Shed package recipes that are not yet in a Conda channel? -********************************************************************************* +******************************************************************************************************** First, you do not need to do anything to your wrapper as long as the package name in the requirement tag matches the name of correct @@ -297,7 +297,7 @@ leave the old versions as they are – simply because of time. 12. What is the recommendation for existing installations? Will I continue to maintain both systems or migrate to the new Conda system eventually? -********************************************************************************* +************************************************************************************************************************************************** Old tools will use the traditional installation system; this system will stay and will be supported for installing old tools to guarantee sustainability @@ -305,7 +305,7 @@ and reproducibility. New tools from the IUC, may be Conda only. 13. What can I do if Conda doesn't work for me? -********************************************************************************* +*********************************************** There is currently a limitation in the way Conda packages are being built. This limitation will be addressed shortly by the Conda community, diff --git a/doc/source/admin/interactive_environments.rst b/doc/source/admin/interactive_environments.rst index ba2aeabb5bb..c333715e1f0 100644 --- a/doc/source/admin/interactive_environments.rst +++ b/doc/source/admin/interactive_environments.rst @@ -61,7 +61,7 @@ Once Node and npm are ready to go, you'll need to install the dependencies Running ``node lib/main.js --help`` should produce some useful help text -.. code-block:: +.. code-block:: console Usage: main [options] @@ -81,7 +81,7 @@ as of 2014. Alternately, the proxy can be stated manually or via a system such a Supervisord. Assuming that the ``$GALAXY_ROOT`` environment variable refers to the location of the Galaxy installation, the command for launching the proxy is: -.. code-block:: console +.. code-block:: console $ node $GALAXY_ROOT/lib/galaxy/web/proxy/js/lib/main.js --ip 0.0.0.0 \ --port 8800 --sessions $GALAXY_ROOT/database/session_map.sqlite \ diff --git a/doc/source/admin/mulled_containers.rst b/doc/source/admin/mulled_containers.rst index 792a47f94ad..e0a5b9129b5 100644 --- a/doc/source/admin/mulled_containers.rst +++ b/doc/source/admin/mulled_containers.rst @@ -1,6 +1,6 @@ -================================= +================================ Containers for Tool Dependencies -================================= +================================ Galaxy tools (also called wrappers) are able to use Conda packages (see more information in our `Galaxy Conda documentation`_) and Docker containers as dependency resolvers. @@ -59,7 +59,7 @@ This will search for containers in the biocontainers organisation. Build all packages from bioconda from the last 24h -^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The BioConda community is building a container for every package they create with a command similar to this. @@ -70,7 +70,7 @@ The BioConda community is building a container for every package they create wit Building Docker containers for local Conda packages -^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Conda packages can be tested with creating a busybox based container for this particular package in the following way. This also demonstrates how you can build a container locally and on-the-fly. @@ -101,7 +101,7 @@ The ``--0`` indicates the build version of the conda package. It is recommended you will override already existing images. For Python Conda packages this extension might look like this ``--py35_1``. Build, test and push a conda-forge package to biocontainers -^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ > You need to have write access to the biocontainers repository diff --git a/doc/source/conf.py b/doc/source/conf.py index 3fc399b5f0a..c9a9f23eb85 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -22,12 +22,10 @@ source_parsers = { '.md': CommonMarkParser, } -####### REQUIRED GALAXY INCLUDES +# REQUIRED GALAXY INCLUDES sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) -####### - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -201,8 +199,8 @@ latex_elements = { # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'Galaxy.tex', u'Galaxy Code Documentation', - u'Galaxy Team', 'manual'), + ('index', 'Galaxy.tex', u'Galaxy Code Documentation', + u'Galaxy Team', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -245,9 +243,9 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'Galaxy', u'Galaxy Code Documentation', - u'Galaxy Team', 'Galaxy', 'Data intensive biology for everyone.', - 'Miscellaneous'), + ('index', 'Galaxy', u'Galaxy Code Documentation', + u'Galaxy Team', 'Galaxy', 'Data intensive biology for everyone.', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. @@ -259,8 +257,8 @@ texinfo_documents = [ # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' -# -- ReadTheDocs.org Settings ------------------------------------------------ +# -- ReadTheDocs.org Settings ------------------------------------------------ class Mock(object): def __init__(self, *args, **kwargs): pass diff --git a/doc/source/dev/build_a_job_runner.rst b/doc/source/dev/build_a_job_runner.rst index dc188f9956c..9e8577c87d7 100644 --- a/doc/source/dev/build_a_job_runner.rst +++ b/doc/source/dev/build_a_job_runner.rst @@ -1,8 +1,8 @@ -BUILD RUNNER FOR GALAXY ------------------------ +Build a job runner +================== -A walk through the steps of building a runner for Galaxy. -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +A walk through the steps of building a runner for Galaxy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In this tutorial, we would build the runner in a block by block fashion (like the building blocks), so we would divide the runner into @@ -14,7 +14,7 @@ To learn more about the basics, please refer to: https://wiki.galaxyproject.org/Admin/GetGalaxy To explore existing runners, please refer to: -https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/jobs/runner +https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/jobs/runners What is required to make a runner for Galaxy? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -51,13 +51,12 @@ Implementation of parent class (galaxy.jobs.runners.\_\_init\_\_.py) - .. rubric:: Class Inheritance structure :name: class-inheritance-structure - .. figure:: https://github.com/varunshankar/galaxy-godocker/raw/master/inherit.png - :alt: + .. image:: inherit.png - .. rubric:: The big picture! :name: the-big-picture-1 - .. figure:: https://raw.githubusercontent.com/varunshankar/galaxy-godocker/master/runner_diag.png + .. image:: runner_diag.png The whole process is divided into different stages for understanding purpose. @@ -218,7 +217,7 @@ Output params: AsynchronousJobState object Without going into much detail, assume there is a queue to track the status of every job. eg: -.. image:: https://raw.githubusercontent.com/varunshankar/galaxy-godocker/master/queue.png +.. image:: queue.png :align: center The galaxy framework updates the status of a job by iterating through the @@ -229,7 +228,7 @@ copy the output files for the completed jobs. Updated result after an iteration (after invocation of check\_watched\_item 6 times): -.. image:: https://raw.githubusercontent.com/varunshankar/galaxy-godocker/master/queue_b.png +.. image:: queue_b.png :align: center diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst index b5e44143c21..504b996e866 100644 --- a/doc/source/dev/index.rst +++ b/doc/source/dev/index.rst @@ -1,15 +1,18 @@ Developer Documentation ======================= -There are two primary areas of documentation developers will be interested in: +.. toctree:: + :maxdepth: 1 + + schema + interactive_environments + build_a_job_runner + faq + +These are other primary areas of documentation developers will be interested in: - `Codebase Documentation`_ for developers of Galaxy - `API Documentation`_ for developers of third party tools interacting with Galaxy -- `Interactive Environments`_ for developers of Galaxy Interactive Environments - -Additionally there is an `FAQ`_ document for new Galaxy developers answering common "How Do I..." questions. .. _Codebase Documentation: ../lib/modules.html .. _API Documentation: ../api_doc.html -.. _Interactive Environments: interactive_environments.html -.. _FAQ: faq.html diff --git a/doc/source/dev/inherit.png b/doc/source/dev/inherit.png new file mode 100644 index 00000000000..671a8d4fb1e Binary files /dev/null and b/doc/source/dev/inherit.png differ diff --git a/doc/source/admin/interactive_environments_success.png b/doc/source/dev/interactive_environments_success.png similarity index 100% rename from doc/source/admin/interactive_environments_success.png rename to doc/source/dev/interactive_environments_success.png diff --git a/doc/source/dev/queue.png b/doc/source/dev/queue.png new file mode 100644 index 00000000000..c4557aa6367 Binary files /dev/null and b/doc/source/dev/queue.png differ diff --git a/doc/source/dev/queue_b.png b/doc/source/dev/queue_b.png new file mode 100644 index 00000000000..1d2c0034532 Binary files /dev/null and b/doc/source/dev/queue_b.png differ diff --git a/doc/source/dev/runner_diag.png b/doc/source/dev/runner_diag.png new file mode 100644 index 00000000000..812b45282b5 Binary files /dev/null and b/doc/source/dev/runner_diag.png differ diff --git a/doc/source/index.rst b/doc/source/index.rst index 71065e61849..cfee8c491c6 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -52,13 +52,12 @@ Indices and tables * :ref:`search` Building this Documentation -========================== +=========================== If you have your own copy of the Galaxy source code, you can also generate your own version of this documentation: :: - $ cd doc - $ make html + $ make -C doc/ html The generated documentation will be in ``doc/build/html/`` and can be viewed with a web browser. Note that you will need to install Sphinx and a fair number of module dependencies before this will produce output. diff --git a/doc/source/lib/galaxy.datatypes.converters.rst b/doc/source/lib/galaxy.datatypes.converters.rst index dacdc91828d..3cbddeed611 100644 --- a/doc/source/lib/galaxy.datatypes.converters.rst +++ b/doc/source/lib/galaxy.datatypes.converters.rst @@ -225,18 +225,18 @@ galaxy.datatypes.converters.vcf_to_vcf_bgzip module :undoc-members: :show-inheritance: -galaxy.datatypes.converters.bcf_bgzip_to_bcf module ---------------------------------------------------- +galaxy.datatypes.converters.bcf_bgzip_to_bcf_converter module +------------------------------------------------------------- -.. automodule:: galaxy.datatypes.converters.bcf_bgzip_to_bcf +.. automodule:: galaxy.datatypes.converters.bcf_bgzip_to_bcf_converter :members: :undoc-members: :show-inheritance: -galaxy.datatypes.converters.bcf_to_bcf_bgzip module ---------------------------------------------------- +galaxy.datatypes.converters.bcf_to_bcf_bgzip_converter module +------------------------------------------------------------- -.. automodule:: galaxy.datatypes.converters.bcf_to_bcf_bgzip +.. automodule:: galaxy.datatypes.converters.bcf_to_bcf_bgzip_converter :members: :undoc-members: :show-inheritance: diff --git a/doc/source/lib/galaxy.jobs.rst b/doc/source/lib/galaxy.jobs.rst index 86c56ad5c51..8905e4691ba 100644 --- a/doc/source/lib/galaxy.jobs.rst +++ b/doc/source/lib/galaxy.jobs.rst @@ -14,7 +14,6 @@ Subpackages galaxy.jobs.actions galaxy.jobs.deferred galaxy.jobs.metrics - galaxy.jobs.rules galaxy.jobs.runners galaxy.jobs.splitters diff --git a/doc/source/lib/galaxy.rst b/doc/source/lib/galaxy.rst index f15c406fbe4..aa8c5fe528e 100644 --- a/doc/source/lib/galaxy.rst +++ b/doc/source/lib/galaxy.rst @@ -22,7 +22,6 @@ Subpackages galaxy.forms galaxy.jobs galaxy.managers - galaxy.metadata galaxy.model galaxy.objectstore galaxy.openid diff --git a/doc/source/lib/galaxy.tools.linters.rst b/doc/source/lib/galaxy.tools.linters.rst index a27e6bf42bd..3463dc953fa 100644 --- a/doc/source/lib/galaxy.tools.linters.rst +++ b/doc/source/lib/galaxy.tools.linters.rst @@ -25,6 +25,14 @@ galaxy.tools.linters.command module :undoc-members: :show-inheritance: +galaxy.tools.linters.general module +----------------------------------- + +.. automodule:: galaxy.tools.linters.general + :members: + :undoc-members: + :show-inheritance: + galaxy.tools.linters.help module -------------------------------- @@ -49,6 +57,14 @@ galaxy.tools.linters.outputs module :undoc-members: :show-inheritance: +galaxy.tools.linters.stdio module +--------------------------------- + +.. automodule:: galaxy.tools.linters.stdio + :members: + :undoc-members: + :show-inheritance: + galaxy.tools.linters.tests module --------------------------------- @@ -57,12 +73,10 @@ galaxy.tools.linters.tests module :undoc-members: :show-inheritance: -galaxy.tools.linters.top_level module +galaxy.tools.linters.xml_order module ------------------------------------- -.. automodule:: galaxy.tools.linters.top_level +.. automodule:: galaxy.tools.linters.xml_order :members: :undoc-members: :show-inheritance: - - diff --git a/doc/source/lib/galaxy.tools.parameters.rst b/doc/source/lib/galaxy.tools.parameters.rst index 28309e13f58..8bfd839859a 100644 --- a/doc/source/lib/galaxy.tools.parameters.rst +++ b/doc/source/lib/galaxy.tools.parameters.rst @@ -65,14 +65,6 @@ galaxy.tools.parameters.meta module :undoc-members: :show-inheritance: -galaxy.tools.parameters.output module -------------------------------------- - -.. automodule:: galaxy.tools.parameters.output - :members: - :undoc-members: - :show-inheritance: - galaxy.tools.parameters.output_collect module --------------------------------------------- @@ -105,4 +97,10 @@ galaxy.tools.parameters.wrapped module :undoc-members: :show-inheritance: +galaxy.tools.parameters.wrapped_json module +------------------------------------------- +.. automodule:: galaxy.tools.parameters.wrapped_json + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/lib/galaxy.util.backports.importlib.rst b/doc/source/lib/galaxy.util.backports.importlib.rst deleted file mode 100644 index 974cf47f98d..00000000000 --- a/doc/source/lib/galaxy.util.backports.importlib.rst +++ /dev/null @@ -1,8 +0,0 @@ -galaxy.util.backports.importlib package -======================================= - -.. automodule:: galaxy.util.backports.importlib - :members: - :undoc-members: - :show-inheritance: - diff --git a/doc/source/lib/galaxy.util.backports.rst b/doc/source/lib/galaxy.util.backports.rst index 69356d80fcd..ca3788ee04b 100644 --- a/doc/source/lib/galaxy.util.backports.rst +++ b/doc/source/lib/galaxy.util.backports.rst @@ -5,11 +5,3 @@ galaxy.util.backports package :members: :undoc-members: :show-inheritance: - -Subpackages ------------ - -.. toctree:: - - galaxy.util.backports.importlib - diff --git a/doc/source/lib/galaxy.util.rst b/doc/source/lib/galaxy.util.rst index dd2eca1c66b..b059a02d85d 100644 --- a/doc/source/lib/galaxy.util.rst +++ b/doc/source/lib/galaxy.util.rst @@ -42,6 +42,14 @@ galaxy.util.bunch module :undoc-members: :show-inheritance: +galaxy.util.checkers module +--------------------------- + +.. automodule:: galaxy.util.checkers + :members: + :undoc-members: + :show-inheritance: + galaxy.util.dbkeys module ------------------------- @@ -50,26 +58,10 @@ galaxy.util.dbkeys module :undoc-members: :show-inheritance: -galaxy.util.debugging module ----------------------------- +galaxy.util.dictifiable module +------------------------------ -.. automodule:: galaxy.util.debugging - :members: - :undoc-members: - :show-inheritance: - -galaxy.util.dictobj module --------------------------- - -.. automodule:: galaxy.util.dictobj - :members: - :undoc-members: - :show-inheritance: - -galaxy.util.directory_hash module ---------------------------------- - -.. automodule:: galaxy.util.directory_hash +.. automodule:: galaxy.util.dictifiable :members: :undoc-members: :show-inheritance: @@ -82,6 +74,14 @@ galaxy.util.expressions module :undoc-members: :show-inheritance: +galaxy.util.filelock module +--------------------------- + +.. automodule:: galaxy.util.filelock + :members: + :undoc-members: + :show-inheritance: + galaxy.util.hash_util module ---------------------------- @@ -98,6 +98,14 @@ galaxy.util.heartbeat module :undoc-members: :show-inheritance: +galaxy.util.image_util module +----------------------------- + +.. automodule:: galaxy.util.image_util + :members: + :undoc-members: + :show-inheritance: + galaxy.util.inflection module ----------------------------- @@ -130,10 +138,10 @@ galaxy.util.lazy_process module :undoc-members: :show-inheritance: -galaxy.util.lrucache module ---------------------------- +galaxy.util.multi_byte module +----------------------------- -.. automodule:: galaxy.util.lrucache +.. automodule:: galaxy.util.multi_byte :members: :undoc-members: :show-inheritance: @@ -178,6 +186,14 @@ galaxy.util.plugin_config module :undoc-members: :show-inheritance: +galaxy.util.postfork module +--------------------------- + +.. automodule:: galaxy.util.postfork + :members: + :undoc-members: + :show-inheritance: + galaxy.util.properties module ----------------------------- @@ -266,6 +282,14 @@ galaxy.util.topsort module :undoc-members: :show-inheritance: +galaxy.util.ucsc module +----------------------- + +.. automodule:: galaxy.util.ucsc + :members: + :undoc-members: + :show-inheritance: + galaxy.util.validation module ----------------------------- diff --git a/doc/source/releases/16.04.rst b/doc/source/releases/16.04.rst index 6f75d92508d..c3bc54c31fc 100644 --- a/doc/source/releases/16.04.rst +++ b/doc/source/releases/16.04.rst @@ -1,9 +1,8 @@ .. to_doc -------------------------------- 16.04 -------------------------------- +=============================== .. announce_start diff --git a/doc/source/releases/16.04_announce.rst b/doc/source/releases/16.04_announce.rst index 779caa179a8..e615714a65a 100644 --- a/doc/source/releases/16.04_announce.rst +++ b/doc/source/releases/16.04_announce.rst @@ -65,6 +65,7 @@ See `our wiki `__ for additio Security =========================================================== + TL;DR **Only Tool Sheds newer than 16.01 should be deployed from now on.** (with commit 449098d8b14b45269be106f6410c0b9145c51d50 from Mar 30 present) @@ -81,7 +82,7 @@ Deprecation Notices =========================================================== API deprecations -~~~~~~~~~~~~~~~~ +---------------- API for history contents, index: * **types**: is no longer a valid parameter but accessible using ``?q=history_content_type&qv=[dataset | dataset_collection]`` @@ -96,7 +97,7 @@ API histories (removed from the available serialized data on all calls): * **state, state_details, state_ids** - can be replaced by specifically requesting a single array of contents, each containing :code:`{ id, state, deleted, visible }` Galaxy no longer on Bitbucket -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------- Galaxy moved its code and development activities from Bitbucket to GitHub in early 2015. Since this time, releases have been mirrored back to Bitbucket. However, after this release, no new changes will be pushed to Bitbucket. Anyone still receiving updates to a Galaxy server via Bitbucket and Mercurial should switch to GitHub and Git. This can be done using the following process: diff --git a/doc/source/releases/16.07.rst b/doc/source/releases/16.07.rst index 35a2577dcd7..911f6d0b029 100644 --- a/doc/source/releases/16.07.rst +++ b/doc/source/releases/16.07.rst @@ -1,9 +1,8 @@ .. to_doc -------------------------------- 16.07 -------------------------------- +=============================== .. announce_start @@ -111,7 +110,7 @@ Enhancements * Fix running tool tests if remote user middleware is enabled. `Pull Request 2173`_ * Improve handling of a missing R environment during Tool Shed dependency - installations. + installations. `Pull Request 2215`_ * Quote some parameters in the trim tool command (thanks to `@nsoranzo `__.) diff --git a/doc/source/releases/index.rst b/doc/source/releases/index.rst index d6b9cfd7d41..5d3859fd31a 100644 --- a/doc/source/releases/index.rst +++ b/doc/source/releases/index.rst @@ -4,6 +4,7 @@ Releases .. toctree:: :maxdepth: 1 + 16.10_announce 16.07_announce 16.04_announce 16.01_announce diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 852020e22dc..187d6192bdb 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1170,10 +1170,13 @@ class OxliCountGraph(OxliBinary): """ OxliCountGraph starts with "OXLI" + one byte version number + 8-bit binary '1' - Test file generated via `load-into-counting.py --n_tables 1 \ - --max-tablesize 1 oxli_countgraph.oxlicg \ - khmer/tests/test-data/100-reads.fq.bz2` + Test file generated via:: + + load-into-counting.py --n_tables 1 --max-tablesize 1 \\ + oxli_countgraph.oxlicg khmer/tests/test-data/100-reads.fq.bz2 + using khmer 2.0 + >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'sequence.csfasta' ) >>> OxliCountGraph().sniff( fname ) @@ -1194,10 +1197,13 @@ class OxliNodeGraph(OxliBinary): """ OxliNodeGraph starts with "OXLI" + one byte version number + 8-bit binary '2' - Test file generated via `load-graph.py --n_tables 1 \ - --max-tablesize 1 oxli_nodegraph.oxling \ - khmer/tests/test-data/100-reads.fq.bz2` + Test file generated via:: + + load-graph.py --n_tables 1 --max-tablesize 1 oxli_nodegraph.oxling \\ + khmer/tests/test-data/100-reads.fq.bz2 + using khmer 2.0 + >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'sequence.csfasta' ) >>> OxliNodeGraph().sniff( fname ) @@ -1218,11 +1224,14 @@ class OxliTagSet(OxliBinary): """ OxliTagSet starts with "OXLI" + one byte version number + 8-bit binary '3' - Test file generated via `load-graph.py --n_tables 1 \ - --max-tablesize 1 oxli_nodegraph.oxling \ - khmer/tests/test-data/100-reads.fq.bz2; \ - mv oxli_nodegraph.oxling.tagset oxli_tagset.oxlits` + Test file generated via:: + + load-graph.py --n_tables 1 --max-tablesize 1 oxli_nodegraph.oxling \\ + khmer/tests/test-data/100-reads.fq.bz2; + mv oxli_nodegraph.oxling.tagset oxli_tagset.oxlits + using khmer 2.0 + >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'sequence.csfasta' ) >>> OxliTagSet().sniff( fname ) @@ -1244,6 +1253,7 @@ class OxliStopTags(OxliBinary): 8-bit binary '4' Test file adapted from khmer 2.0's "khmer/tests/test-data/goodversion-k32.stoptags" + >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'sequence.csfasta' ) >>> OxliStopTags().sniff( fname ) @@ -1264,11 +1274,14 @@ class OxliSubset(OxliBinary): """ OxliSubset starts with "OXLI" + one byte version number + 8-bit binary '5' - Test file generated via `load-graph.py -k 20 example \ - tests/test-data/random-20-a.fa; \ - partition-graph.py example; \ - mv example.subset.0.pmap oxli_subset.oxliss` + Test file generated via:: + + load-graph.py -k 20 example tests/test-data/random-20-a.fa; + partition-graph.py example; + mv example.subset.0.pmap oxli_subset.oxliss + using khmer 2.0 + >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'sequence.csfasta' ) >>> OxliSubset().sniff( fname ) @@ -1288,11 +1301,15 @@ class OxliGraphLabels(OxliBinary): """ OxliGraphLabels starts with "OXLI" + one byte version number + 8-bit binary '6' - Test file generated via `python -c "from khmer import GraphLabels; \ - gl = GraphLabels(20, 1e7, 4); gl.consume_fasta_and_tag_with_labels( - 'tests/test-data/test-labels.fa'); \ - gl.save_labels_and_tags('oxli_graphlabels.oxligl')"` + Test file generated via:: + + python -c "from khmer import GraphLabels; \\ + gl = GraphLabels(20, 1e7, 4); \\ + gl.consume_fasta_and_tag_with_labels('tests/test-data/test-labels.fa'); \\ + gl.save_labels_and_tags('oxli_graphlabels.oxligl')" + using khmer 2.0 + >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( 'sequence.csfasta' ) >>> OxliGraphLabels().sniff( fname ) diff --git a/lib/galaxy/datatypes/checkers.py b/lib/galaxy/datatypes/checkers.py index 9fc5f098586..ab2fce29024 100644 --- a/lib/galaxy/datatypes/checkers.py +++ b/lib/galaxy/datatypes/checkers.py @@ -1,4 +1,4 @@ -"""Module proxies :module:`galaxy.util.checkers` for backward compatibility. +"""Module proxies :mod:`galaxy.util.checkers` for backward compatibility. External datatypes may make use of these functions. """ diff --git a/lib/galaxy/datatypes/constructive_solid_geometry.py b/lib/galaxy/datatypes/constructive_solid_geometry.py index 8e58c9e85d8..43f1747d2e7 100644 --- a/lib/galaxy/datatypes/constructive_solid_geometry.py +++ b/lib/galaxy/datatypes/constructive_solid_geometry.py @@ -134,7 +134,7 @@ Binary.register_sniffable_binary_format("plybinary", "plybinary", PlyBinary) class Vtk(object): - """ + r""" The Visualization Toolkit provides a number of source and writer objects to read and write popular data file formats. The Visualization Toolkit also provides some of its own file formats. @@ -151,8 +151,8 @@ class Vtk(object): i.e., the numbers that define points coordinates, scalars, cell indices, and so forth. - Binary data must be placed into the file immediately after the newline (\n) - character from the previous ASCII keyword and parameter sequence. + Binary data must be placed into the file immediately after the newline + ('\\n') character from the previous ASCII keyword and parameter sequence. TODO: only legacy formats are currently supported and support for XML formats should be added. diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 6bc7e2a7f8b..0fbcaaaf07a 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -66,14 +66,13 @@ class Data( object ): 'test' >>> type( DataTest.metadata_spec.test.param ) - """ edam_data = "data_0006" edam_format = "format_1915" # Data is not chunkable by default. CHUNKABLE = False - #: dictionary of metadata fields for this datatype:: + #: Dictionary of metadata fields for this datatype metadata_spec = None # Add metadata elements @@ -855,7 +854,7 @@ class Text( Data ): @dataproviders.decorators.dataprovider_factory( 'line', dataproviders.line.FilteredLineDataProvider.settings ) def line_dataprovider( self, dataset, **settings ): """ - Returns an iterator over the dataset's lines (that have been `strip`ed) + Returns an iterator over the dataset's lines (that have been stripped) optionally excluding blank lines and lines that start with a comment character. """ dataset_source = dataproviders.dataset.DatasetDataProvider( dataset ) @@ -962,10 +961,9 @@ def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skip """ Returns the first LINE_COUNT lines wrapped to WIDTH - ## >>> fname = get_test_fname('4.bed') - ## >>> get_file_peek(fname) - ## 'chr22 30128507 31828507 uc003bnx.1_cds_2_0_chr22_29227_f 0 +\n' - + >>> fname = get_test_fname('4.bed') + >>> get_file_peek(fname, LINE_COUNT=1) + u'chr22\\t30128507\\t31828507\\tuc003bnx.1_cds_2_0_chr22_29227_f\\t0\\t+\\n' """ # Set size for file.readline() to a negative number to force it to # read until either a newline or EOF. Needed for datasets with very diff --git a/lib/galaxy/datatypes/dataproviders/__init__.py b/lib/galaxy/datatypes/dataproviders/__init__.py index ab97e39582c..caf9a7e328c 100644 --- a/lib/galaxy/datatypes/dataproviders/__init__.py +++ b/lib/galaxy/datatypes/dataproviders/__init__.py @@ -6,7 +6,6 @@ consumer datum by datum. As well as subclassing and overriding to get the proper data, Dataproviders can be piped from one to the other. -..example:: .. note:: be careful to NOT pipe providers into subclasses of those providers. Subclasses provide all the functionality of their superclasses, diff --git a/lib/galaxy/datatypes/dataproviders/column.py b/lib/galaxy/datatypes/dataproviders/column.py index d57f415026c..bc97fc6f699 100644 --- a/lib/galaxy/datatypes/dataproviders/column.py +++ b/lib/galaxy/datatypes/dataproviders/column.py @@ -78,8 +78,8 @@ class ColumnarDataProvider( line.RegexLineDataProvider ): Optional: defaults to the tab character. :type deliminator: str - .. note: that the subclass constructors are passed kwargs - so they're - params (limit, offset, etc.) are also applicable here. + .. note:: that the subclass constructors are passed kwargs - so they're + params (limit, offset, etc.) are also applicable here. """ # TODO: other columnar formats: csv, etc. super( ColumnarDataProvider, self ).__init__( source, **kwargs ) @@ -141,9 +141,13 @@ class ColumnarDataProvider( line.RegexLineDataProvider ): The function will compare the column at index `column` against `val` using the given op where op is one of: - lt: less than, le: less than or equal to, - eq: equal to, ne: not equal to, - ge: greather than or equal to, gt: greater than + + - lt: less than + - le: less than or equal to + - eq: equal to + - ne: not equal to + - ge: greather than or equal to + - gt: greater than `val` is cast as float here and will return None if there's a parsing error. """ @@ -173,9 +177,10 @@ class ColumnarDataProvider( line.RegexLineDataProvider ): The function will compare the column at index `column` against `val` using the given op where op is one of: - eq: exactly matches, - has: the column contains the substring `val`, - re: the column matches the regular expression in `val` + + - eq: exactly matches + - has: the column contains the substring `val` + - re: the column matches the regular expression in `val` """ if 'eq' == op: return lambda d: d[column] == val @@ -195,8 +200,9 @@ class ColumnarDataProvider( line.RegexLineDataProvider ): The function will compare the column at index `column` against `val` using the given op where op is one of: - eq: the list `val` exactly matches the list in the column, - has: the list in the column contains the sublist `val`, + + - eq: the list `val` exactly matches the list in the column + - has: the list in the column contains the sublist `val` """ if 'eq' == op: val = self.parse_value( val, 'list' ) @@ -210,8 +216,9 @@ class ColumnarDataProvider( line.RegexLineDataProvider ): Return parser dictionary keyed for each columnar type (as defined in datatypes). - .. note: primitives only by default (str, int, float, boolean, None). + .. note:: primitives only by default (str, int, float, boolean, None). Other (more complex) types are retrieved as strings. + :returns: a dictionary of the form: `{ : }` """ @@ -324,8 +331,8 @@ class DictDataProvider( ColumnarDataProvider ): A combination use of both `column_names` and `indeces` allows 'picking' key/value pairs from the source. - .. note: that the subclass constructors are passed kwargs - so they're - params (limit, offset, etc.) are also applicable here. + .. note:: The subclass constructors are passed kwargs - so their + params (limit, offset, etc.) are also applicable here. """ settings = { 'column_names' : 'list:str', diff --git a/lib/galaxy/datatypes/dataproviders/dataset.py b/lib/galaxy/datatypes/dataproviders/dataset.py index 6547854567d..2ed8314fa48 100644 --- a/lib/galaxy/datatypes/dataproviders/dataset.py +++ b/lib/galaxy/datatypes/dataproviders/dataset.py @@ -56,6 +56,7 @@ class DatasetDataProvider( base.DataProvider ): def get_column_metadata_from_dataset( cls, dataset ): """ Convenience class method to get column metadata from a dataset. + :returns: a dictionary of `column_count`, `column_types`, and `column_names` if they're available, setting each to `None` if not. """ @@ -69,6 +70,7 @@ class DatasetDataProvider( base.DataProvider ): def get_metadata_column_types( self, indeces=None ): """ Return the list of `column_types` for this dataset or `None` if unavailable. + :param indeces: the indeces for the columns of which to return the types. Optional: defaults to None (return all types) :type indeces: list of ints @@ -88,6 +90,7 @@ class DatasetDataProvider( base.DataProvider ): def get_metadata_column_names( self, indeces=None ): """ Return the list of `column_names` for this dataset or `None` if unavailable. + :param indeces: the indeces for the columns of which to return the names. Optional: defaults to None (return all names) :type indeces: list of ints @@ -108,8 +111,10 @@ class DatasetDataProvider( base.DataProvider ): def get_indeces_by_column_names( self, list_of_column_names ): """ Return the list of column indeces when given a list of column_names. + :param list_of_column_names: the names of the columns of which to get indeces. :type list_of_column_names: list of strs + :raises KeyError: if column_names are not found :raises ValueError: if an entry in list_of_column_names is not in column_names """ @@ -399,7 +404,8 @@ class IntervalDataProvider( column.ColumnarDataProvider ): # WITHOUT reading the entire seq into memory - possibly apply some version of limit/offset class FastaDataProvider( base.FilteredDataProvider ): """ - Class that returns fasta format data in a list of maps of the form: + Class that returns fasta format data in a list of maps of the form:: + { id: , sequence: @@ -432,7 +438,8 @@ class FastaDataProvider( base.FilteredDataProvider ): class TwoBitFastaDataProvider( DatasetDataProvider ): """ - Class that returns fasta format data in a list of maps of the form: + Class that returns fasta format data in a list of maps of the form:: + { id: , sequence: diff --git a/lib/galaxy/datatypes/dataproviders/decorators.py b/lib/galaxy/datatypes/dataproviders/decorators.py index a7a662621a7..99e01e1934a 100644 --- a/lib/galaxy/datatypes/dataproviders/decorators.py +++ b/lib/galaxy/datatypes/dataproviders/decorators.py @@ -32,25 +32,24 @@ def has_dataproviders( cls ): in the class. This allows a class to maintain a name -> method map, effectively - 'registering' dataprovider factory methods. + 'registering' dataprovider factory methods:: - .. example:: - @has_dataproviders - class MyDtype( data.Data ): + @has_dataproviders + class MyDtype( data.Data ): - @dataprovider_factory( 'bler' ) - def provide_some_bler( self, dataset, **settings ): - '''blerblerbler''' - dataset_source = providers.DatasetDataProvider( dataset ) - # ... chain other, intermidiate providers here - return providers.BlerDataProvider( dataset_source, **settings ) + @dataprovider_factory( 'bler' ) + def provide_some_bler( self, dataset, **settings ): + '''blerblerbler''' + dataset_source = providers.DatasetDataProvider( dataset ) + # ... chain other, intermidiate providers here + return providers.BlerDataProvider( dataset_source, **settings ) - # use the base method in data.Data - provider = dataset.datatype.dataprovider( dataset, 'bler', - my_setting='blah', ... ) - # OR directly from the map - provider = dataset.datatype.dataproviders[ 'bler' ]( dataset, - my_setting='blah', ... ) + # use the base method in data.Data + provider = dataset.datatype.dataprovider( dataset, 'bler', + my_setting='blah', ... ) + # OR directly from the map + provider = dataset.datatype.dataproviders[ 'bler' ]( dataset, + my_setting='blah', ... ) """ # init the class dataproviders map if necc. if not hasattr( cls, _DATAPROVIDER_CLASS_MAP_KEY ): @@ -82,15 +81,15 @@ def dataprovider_factory( name, settings=None ): function to parse query strings to __init__ arguments as the `parse_query_string_settings` attribute of the factory function. - An example use of the `parse_query_string_settings`: - ..example:: - kwargs = dataset.datatype.dataproviders[ provider ].parse_query_string_settings( query_kwargs ) - return list( dataset.datatype.dataprovider( dataset, provider, **kwargs ) ) + An example use of the `parse_query_string_settings`:: + + kwargs = dataset.datatype.dataproviders[ provider ].parse_query_string_settings( query_kwargs ) + return list( dataset.datatype.dataprovider( dataset, provider, **kwargs ) ) :param name: what name/key to register the factory under in `cls.dataproviders` :type name: any hashable var :param settings: dictionary containing key/type pairs for parsing query strings - to __init__ arguments + to __init__ arguments :type settings: dictionary """ # TODO:?? use *args for settings allowing mulitple dictionaries diff --git a/lib/galaxy/datatypes/dataproviders/line.py b/lib/galaxy/datatypes/dataproviders/line.py index 7836bf2863f..0809773450e 100644 --- a/lib/galaxy/datatypes/dataproviders/line.py +++ b/lib/galaxy/datatypes/dataproviders/line.py @@ -88,7 +88,7 @@ class RegexLineDataProvider( FilteredLineDataProvider ): of regexs. .. note:: the regex matches are effectively OR'd (if **any** regex matches - the line it is considered valid and will be provided). + the line it is considered valid and will be provided). """ settings = { 'regex_list' : 'list:escaped', diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index c977d8fd288..d1660145064 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -1007,19 +1007,28 @@ class DotBracket ( Sequence ): Galaxy Dbn (Dot-Bracket notation) rules: * The first non-empty line is a header line: no comment lines are allowed. + * A header line starts with a '>' symbol and continues with 0 or multiple symbols until the line ends. + * The second non-empty line is a sequence line. - * A sequence line may only include chars that match the Fasta format (https://en.wikipedia.org/wiki/FASTA_format#Sequence_representation) symbols for nucleotides: ACGTURYKMSWBDHVN, and may thus not include whitespaces. + + * A sequence line may only include chars that match the FASTA format (https://en.wikipedia.org/wiki/FASTA_format#Sequence_representation) symbols for nucleotides: ACGTURYKMSWBDHVN, and may thus not include whitespaces. * A sequence line has no prefix and no suffix. * A sequence line is case insensitive. + * The third non-empty line is a structure (Dot-Bracket) line and only describes the 2D structure of the sequence above it. + * A structure line must consist of the following chars: '.{}[]()'. * A structure line must be of the same length as the sequence line, and each char represents the structure of the nucleotide above it. * A structure line has no prefix and no suffix. * A nucleotide pairs with only 1 or 0 other nucleotides. + * In a structure line, the number of '(' symbols equals the number of ')' symbols, the number of '[' symbols equals the number of ']' symbols and the number of '{' symbols equals the number of '}' symbols. + * The format accepts multiple entries per file, given that each entry is provided as three lines: the header, sequence and structure line. + * Sniffing is only applied on the first entry. + * Empty lines are allowed. """ diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 2a4c580d111..b52a1e491a0 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -1058,31 +1058,31 @@ class ConnectivityTable( Tabular ): The ConnectivityTable (CT) is a file format used for describing RNA 2D structures by tools including MFOLD, UNAFOLD and the RNAStructure package. The tabular file format is defined as - follows: + follows:: -5 energy = -12.3 sequence name -1 G 0 2 0 1 -2 A 1 3 0 2 -3 A 2 4 0 3 -4 A 3 5 0 4 -5 C 4 6 1 5 + 5 energy = -12.3 sequence name + 1 G 0 2 0 1 + 2 A 1 3 0 2 + 3 A 2 4 0 3 + 4 A 3 5 0 4 + 5 C 4 6 1 5 The links given at the edam ontology page do not indicate what type of separator is used (space or tab) while different implementations exist. The implementation that uses spaces as - separator (implemented in RNAStructure) is as follows: + separator (implemented in RNAStructure) is as follows:: - 10 ENERGY = -34.8 seqname - 1 G 0 2 9 1 - 2 G 1 3 8 2 - 3 G 2 4 7 3 - 4 a 3 5 0 4 - 5 a 4 6 0 5 - 6 a 5 7 0 6 - 7 C 6 8 3 7 - 8 C 7 9 2 8 - 9 C 8 10 1 9 - 10 a 9 0 0 10 + 10 ENERGY = -34.8 seqname + 1 G 0 2 9 1 + 2 G 1 3 8 2 + 3 G 2 4 7 3 + 4 a 3 5 0 4 + 5 a 4 6 0 5 + 6 a 5 7 0 6 + 7 C 6 8 3 7 + 8 C 7 9 2 8 + 9 C 8 10 1 9 + 10 a 9 0 0 10 """ i = 0 diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index f87e6844bbe..4aa18d54c11 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -541,9 +541,8 @@ class JobConfiguration( object ): a list of IDs, the JobToolConfigurations for the first id in ``ids`` matching a tool definition. - .. note:: - - You should not mix tool shed tool IDs, versionless tool shed IDs, and tool config tool IDs that refer to the same tool. + .. note:: You should not mix tool shed tool IDs, versionless tool shed + IDs, and tool config tool IDs that refer to the same tool. :param ids: Tool ID or IDs to fetch the JobToolConfiguration of. :type ids: list or str. diff --git a/lib/galaxy/jobs/metrics/instrumenters/__init__.py b/lib/galaxy/jobs/metrics/instrumenters/__init__.py index 0a81cffb56b..b996daf5c79 100644 --- a/lib/galaxy/jobs/metrics/instrumenters/__init__.py +++ b/lib/galaxy/jobs/metrics/instrumenters/__init__.py @@ -1,4 +1,4 @@ -"""This module describes the abstract interface for :class:`InstrumentPlugin`s. +"""This module describes the abstract interface for :class:`InstrumentPlugin`. These are responsible for collecting and formatting a coherent set of metrics. """ diff --git a/lib/galaxy/managers/__init__.py b/lib/galaxy/managers/__init__.py index d5a7ab9252e..fc897383194 100644 --- a/lib/galaxy/managers/__init__.py +++ b/lib/galaxy/managers/__init__.py @@ -6,26 +6,26 @@ Encapsulates the intersection of trans (or trans.sa_session), models, and Controllers. Responsibilities: - model operations that involve the trans/sa_session (CRUD) - security: - ownership, accessibility - common aspect-oriented operations via new mixins: - sharable, annotatable, tagable, ratable + +- model operations that involve the trans/sa_session (CRUD) +- security: ownership, accessibility +- common aspect-oriented operations via new mixins: sharable, annotatable, + tagable, ratable Not responsible for: - encoding/decoding ids - any http gobblygook - formatting of returned data (always python structures) - formatting of raised errors + +- encoding/decoding ids +- any http gobblygook +- formatting of returned data (always python structures) +- formatting of raised errors The goal is to have Controllers only handle: - query-string/payload parsing and encoding/decoding ids - http - return formatting -and: - control, improve namespacing in Controllers - DRY for Controller ops (define here - use in both UI/API Controllers) +- query-string/payload parsing and encoding/decoding ids +- http +- return formatting +- control, improve namespacing in Controllers +- DRY for Controller ops (define here - use in both UI/API Controllers) In other words, 'Business logic' independent of web transactions/user context (trans) should be pushed into models - but logic that requires the context diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py index 73bf377cd8a..afe2ce2db76 100644 --- a/lib/galaxy/managers/base.py +++ b/lib/galaxy/managers/base.py @@ -4,9 +4,10 @@ defines a base ModelManager, ModelSerializer, and ModelDeserializer. ModelManagers are used for operations on models that occur outside the scope of a single model object, such as: - - object creation - - object lookup - - interactions between 2+ objects of different model classes + +- object creation +- object lookup +- interactions between 2+ objects of different model classes (Since these were to replace model Mixins from web/framework/base/controller.py the rule of thumb used there also generally @@ -903,10 +904,12 @@ class ModelFilterParser( HasAModelManager ): """ Converts string tuples (partially converted query string params) of attr, op, val into either: - - ORM based filters (filters that can be applied by the ORM at the SQL - level) or - - functional filters (filters that use derived values or values not - within the SQL tables) + + - ORM based filters (filters that can be applied by the ORM at the SQL + level) or + - functional filters (filters that use derived values or values not + within the SQL tables) + These filters can then be applied to queries. This abstraction allows 'smarter' application of limit and offset at either the diff --git a/lib/galaxy/managers/datasets.py b/lib/galaxy/managers/datasets.py index 812d809e7bb..6dd1d402f97 100644 --- a/lib/galaxy/managers/datasets.py +++ b/lib/galaxy/managers/datasets.py @@ -236,7 +236,8 @@ class DatasetDeserializer( base.ModelDeserializer, deletable.PurgableDeserialize def deserialize_permissions( self, dataset, key, permissions, user=None, **context ): """ Create permissions for each list of encoded role ids in the (validated) - `permissions` dictionary, where `permissions` is in the form: + `permissions` dictionary, where `permissions` is in the form:: + { 'manage': [ , ... ], 'access': [ , ... ] } """ self.manager.permissions.manage.error_unless_permitted( dataset, user ) @@ -355,7 +356,7 @@ class DatasetAssociationManager( base.ModelManager, """ Return True if this hda/ldda is a composite type dataset. - .. note: see also (whereever we keep information on composite datatypes?) + .. note:: see also (whereever we keep information on composite datatypes?) """ return dataset_assoc.extension in self.app.datatypes_registry.get_composite_extensions() diff --git a/lib/galaxy/managers/libraries.py b/lib/galaxy/managers/libraries.py index 3fb32a2cb54..9bcb4d82fd6 100644 --- a/lib/galaxy/managers/libraries.py +++ b/lib/galaxy/managers/libraries.py @@ -33,7 +33,7 @@ class LibraryManager( object ): :type check_accessible: bool :returns: the requested library - :rtype: Library + :rtype: galaxy.model.Library """ try: library = trans.sa_session.query( trans.app.model.Library ).filter( trans.app.model.Library.table.c.id == decoded_library_id ).one() @@ -145,8 +145,8 @@ class LibraryManager( object ): """ Check if library is accessible to user. - :param folder: library - :type folder: Library + :param library: library + :type library: galaxy.model.Library :param check_accessible: flag whether to check that user can access library :type check_accessible: bool @@ -176,7 +176,7 @@ class LibraryManager( object ): Return library data in the form of a dictionary. :param library: library - :type library: Library + :type library: galaxy.model.Library :returns: dict with data about the library :rtype: dictionary @@ -201,7 +201,7 @@ class LibraryManager( object ): Load all permissions currently related to the given library. :param library: the model object - :type library: Library + :type library: galaxy.model.Library :rtype: dictionary :returns: dict of current roles for all available permission types diff --git a/lib/galaxy/managers/rbac_secured.py b/lib/galaxy/managers/rbac_secured.py index e90a50e2959..0b2883bba78 100644 --- a/lib/galaxy/managers/rbac_secured.py +++ b/lib/galaxy/managers/rbac_secured.py @@ -13,7 +13,7 @@ class RBACPermissionFailedException( galaxy.exceptions.InsufficientPermissionsEx class RBACPermission( object ): """ - Base class for wrangling/controlling the permissions ORM models (*Permissions, Roles) + Base class for wrangling/controlling the permissions ORM models (\*Permissions, Roles) that control which users can perform certain actions on their associated models (Libraries, Datasets). """ @@ -68,8 +68,9 @@ class DatasetRBACPermission( RBACPermission ): DatasetPermissions are typed (but not polymorphic themselves) by a string 'action'. There are two types: - - manage permissions : can a role manage the permissions on a dataset - - access : can a role read/look at/copy a dataset + + - manage permissions : can a role manage the permissions on a dataset + - access : can a role read/look at/copy a dataset """ permissions_class = model.DatasetPermissions action_name = None diff --git a/lib/galaxy/managers/roles.py b/lib/galaxy/managers/roles.py index 3bd298b723a..085594c5851 100644 --- a/lib/galaxy/managers/roles.py +++ b/lib/galaxy/managers/roles.py @@ -33,7 +33,7 @@ class RoleManager( base.ModelManager ): :type decoded_role_id: int :returns: the loaded Role object - :rtype: Role + :rtype: galaxy.model.Role :raises: InconsistentDatabase, RequestParameterInvalidException, InternalServerError """ diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index fbb16e2d283..5076dd8bd56 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -268,6 +268,7 @@ class TextToolParameter( ToolParameter ): class IntegerToolParameter( TextToolParameter ): """ Parameter that takes an integer value. + >>> from galaxy.util.bunch import Bunch >>> trans = Bunch( history=Bunch(), workflow_building_mode=True ) >>> p = IntegerToolParameter( None, XML( '' ) ) @@ -343,6 +344,7 @@ class IntegerToolParameter( TextToolParameter ): class FloatToolParameter( TextToolParameter ): """ Parameter that takes a real number value. + >>> from galaxy.util.bunch import Bunch >>> trans = Bunch( history=Bunch(), workflow_building_mode=True ) >>> p = FloatToolParameter( None, XML( '' ) ) diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index f0a39ce3be5..2697dfd1cfd 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -74,7 +74,7 @@ the tool menu immediately following the hyperlink for the tool (based on the - tag sets. Any number of tests can be included, and each test is wrapped within separate tag sets. Functional tests are -executed via [Planemo](http://planemo.readthedocs.io/) or the +executed via [Planemo](https://planemo.readthedocs.io/) or the [run_tests.sh](https://github.com/galaxyproject/galaxy/blob/dev/run_tests.sh) shell script distributed with Galaxy. The documentation contained here is mostly reference documentation, for tutorials on writing tool tests please check out Planemo's -[Test-Driven Development](http://planemo.readthedocs.io/en/latest/writing_advanced.html#test-driven-development) +[Test-Driven Development](https://planemo.readthedocs.io/en/latest/writing_advanced.html#test-driven-development) documentation or the much older wiki content for [WritingTests](https://wiki.galaxyproject.org/Admin/Tools/WritingTests). @@ -928,7 +928,7 @@ associated input ``repeat``. @@ -1216,7 +1216,7 @@ This directive specifies a test for an output's discovered dataset. It acts as a ### Example The functional test tool -multi_output_assign_primary.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/multi_output_assign_primary.xml) +[multi_output_assign_primary.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/multi_output_assign_primary.xml) provides a demonstration of using this tag. ```xml @@ -1299,7 +1299,7 @@ tool demonstrates basic usage of an ``output_collection`` test expectation. The [CWPair2](https://github.com/galaxyproject/tools-iuc/blob/master/tools/cwpair2/cwpair2.xml) tool demonstrates that ``element``s can specify a ``compare`` attribute just -like [``output``](#tool|tests|test|output). +like [output](#tool-tests-test-output). ```xml @@ -1326,7 +1326,7 @@ test tool demonstrates the use of nested ``element`` directives as described above. Notice also that it tests the output with ``assert_contents`` instead of supplying a ``file`` attribute. Like hinted at with with ``compare`` attribute above, the ``element`` tag can specify any of the test attributes that apply to -the [``output``](#tool|tests|test|output) (e.g. ``md5``, ``compare``, ``diff``, +the [output](#tool-tests-test-output) (e.g. ``md5``, ``compare``, ``diff``, etc...). ```xml @@ -1412,7 +1412,7 @@ many of the default assertion tags that come with Galaxy and examples of each can be found below. The implementation of these tags are simply Python functions defined in the -[``galaxy.tools.verify.asserts``](https://github.com/galaxyproject/galaxy/tree/dev/lib/galaxy/tools/verify/asserts) +[galaxy.tools.verify.asserts](https://github.com/galaxyproject/galaxy/tree/dev/lib/galaxy/tools/verify/asserts) module. ]]> @@ -1511,7 +1511,7 @@ module. `` tag within the ```` tag set -maps to a command line parameter within the [``command``](#tool|command) tag. Most +maps to a command line parameter within the [command](#tool-command) tag. Most tools will not need to specify any attributes on this tag itself.]]> @@ -1602,7 +1602,7 @@ statement. A good example tool that demonstrates many conditional parameters is ``` -The first directive following the conditional is a [``param``](#tool|inputs|param), +The first directive following the conditional is a [param](#tool-inputs-param), this param must be of type ``select`` or ``boolean``. Depending on the value a user selects for this "test" parameter - different UI elements will be shown. These different paths are described by the following the ``when`` blocks shown @@ -1705,7 +1705,7 @@ Is referenced parameter is the same group.]]> This directive describes one potential set of input for the tool at this depth. See documentation for the -[``conditional``](#tool|inputs|conditional) block for more details and examples (XML +[conditional](#tool-inputs-conditional) block for more details and examples (XML and corresponding Cheetah conditionals). @@ -1779,7 +1779,7 @@ This is an example test case with multiple repeat elements for the example above ``` -See the documentation on the [``repeat`` test directive](#tool|tests|test|repeat). +See the documentation on the [repeat test directive](#tool-tests-test-repeat). An older way to specify repeats in a test is by instances that are created by referring to names with a special format: "_|" @@ -1904,7 +1904,7 @@ parameter being described. All the attributes for the ``param`` element are documented below for completeness, but here are the common ones for each type are as follows: -$attribute_list:name,type,optional,label,help,argument +$attribute_list:name,type,optional,label,help,argument:4 ### Parameter Types @@ -1913,8 +1913,6 @@ $attribute_list:name,type,optional,label,help,argument When ``type="text"``, the parameter is free form text and appears as a text box in the tool form. -$attribute_list:value,size,area - ##### Examples Sometimes you need labels for data or graph axes, chart titles, etc. This can be @@ -1932,23 +1930,25 @@ rendered on the tool form as a text area instead of a single line text box. ``` +$attribute_list:value,size,area:5 + #### ``integer`` and ``float`` These parameters represent whole number and real numbers, respectively. -$attribute_list:value,min,max - ##### Example ``` ``` +$attribute_list:value,min,max:5 + #### ``boolean`` This represents a binary true or false value. -$attribute_list:checked,truevalue,falsevalue +$attribute_list:checked,truevalue,falsevalue:5 #### ``data`` @@ -1998,10 +1998,10 @@ Some example tools using ``multiple="true"`` data parameters include: - [multi_data_optional.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/multi_data_optional.xml) Additionally, a detailed discussion of handling multiple homogenous files can be found in the -the [Planemo Documentation](http://planemo.readthedocs.io/en/latest/writing_advanced.html#consuming-collections) +the [Planemo Documentation](https://planemo.readthedocs.io/en/latest/writing_advanced.html#consuming-collections) on this topic. -$attribute_list:format,multiple +$attribute_list:format,multiple:5 #### ``select`` @@ -2027,22 +2027,20 @@ value of ``$upstream_or_down`` will be ``d``, ``u``, ``u,d``, or "". ``` -$attribute_list:data_ref,dynamic_options,display,multiple +$attribute_list:data_ref,dynamic_options,display,multiple:5 #### ``data_column`` This parameter type is used to select columns from a parameter. -$attribute_list:force_select,numerical,use_header_name +$attribute_list:force_select,numerical,use_header_name:5 #### ``drill_down`` -$attribute_list:hierarchy +$attribute_list:hierarchy:5 #### ``data_collection`` -$attribute_list:format,collection_type - The following will create a parameter that only accepts paired FASTQ files grouped into a collection. ##### Examples @@ -2053,12 +2051,12 @@ The following will create a parameter that only accepts paired FASTQ files group ``` More detailed information on writing tools that consume collections can be found -in the [planemo documentation](http://planemo.readthedocs.io/en/latest/writing_advanced.html#collections). +in the [planemo documentation](https://planemo.readthedocs.io/en/latest/writing_advanced.html#collections). + +$attribute_list:format,collection_type:5 #### ``color`` -$attribute_list:value,rgb - ##### Examples The following example will create a color selector parameter. @@ -2081,6 +2079,8 @@ sanitizer to prevent Galaxy from escaping the result. ``` +$attribute_list:value,rgb:5 + This covers examples of the most common parameter types, the remaining parameter types are more obsecure and less likely to be useful for most tool authors. @@ -2371,12 +2371,12 @@ parameters supplied in the form with the actual tool executable). Any word inside it starting with a dollar sign (``$``) will be treated as a variable whose values can be acquired from one of three sources: parameters, metadata, or output files. After the substitution of variables with their values, the content -is interpreted with [Cheetah](http://www.cheetahtemplate.org/) and finally given +is interpreted with [Cheetah](https://pythonhosted.org/Cheetah/) and finally given to the interpreter specified in the corresponding attribute (if any). ### Examples -The following uses a compiled executable ([bedtools](http://bedtools.readthedocs.io/en/latest/)). +The following uses a compiled executable ([bedtools](https://bedtools.readthedocs.io/en/latest/)). ```xml bed12ToBed6 -i '$input' > '$output' @@ -2442,7 +2442,7 @@ the following feature components. A set of "metadata information" is defined for each supported data type (see the ``MetadataElement`` objects in the various data types classes in -[``/lib/galaxy/datatypes``](https://github.com/galaxyproject/galaxy/tree/dev/lib/galaxy/datatypes). +[/lib/galaxy/datatypes](https://github.com/galaxyproject/galaxy/tree/dev/lib/galaxy/datatypes). The ``DatasetFilenameWrapper`` class in the [/lib/galaxy/tools/wrappers.py](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/tools/wrappers.py) code file wraps a metadata collection to return metadata parameters wrapped @@ -2488,8 +2488,6 @@ In additon to demonstrating accessing metadata, this example demonstrates: * ``#set datatype = $input.datatype`` which is the syntax for defining variables in Cheetah. - - ### Reserved Variables Galaxy provides a few pre-defined variables which can be used in your command line, @@ -2514,7 +2512,7 @@ Name | Description ---- | ----------- ``\${GALAXY_SLOTS:-4}`` | Number of cores/threads allocated by the job runner or resource manager to the tool for the given job (here 4 is the default number of threads to use if running via custom runner that does not configure GALAXY_SLOTS or in an older Galaxy runtime). -See the [Planemo docs](http://planemo.readthedocs.io/en/latest/writing_advanced.html#cluster-usage) +See the [Planemo docs](https://planemo.readthedocs.io/en/latest/writing_advanced.html#cluster-usage) on the topic of ``GALAXY_SLOTS`` for more information and examples. ### Attributes @@ -2578,7 +2576,7 @@ deprecated and using the ``$__tool_directory__`` variable is superior. See [/tools/filters/sorter.xml](https://github.com/galaxyproject/galaxy/blob/master/tools/filters/sorter.xml) for typical examples of how to use this tag set. This directive is used to described static lists of options and is contained -within the [``param``](#tool|inputs|param) directive when the ``type`` attribute +within the [param](#tool-inputs-param) directive when the ``type`` attribute value is ``select`` (i.e. ````). ### Example @@ -2655,7 +2653,7 @@ exclusively use ``filter``s to populate options. * ``from_parameter`` - The options for the select list are dynamically obtained from a parameter. * Using ``filter``s - various filters can be used to populate options, see - examples in the [``filter``](#tool|inputs|param|options|filter) documentation. + examples in the [filter](#tool-inputs-param-options-filter) documentation. ### ``from_data_table`` @@ -2723,7 +2721,7 @@ and demonstrates generating options from a dataset directly. Filters can be used to generate options from dataset directly also as the example below demonstrates (many more examples are present in the -[``filter``](#tool|inputs|param|options|filter) documentation). +[filter](#tool-inputs-param-options-filter) documentation). ```xml @@ -2741,7 +2739,7 @@ dataset is available, admins must add the database to the local file named "blastdb.loc". All such databases in that file are included in the options of the select list. For a local instance, the file (e.g. ``blastdb.loc`` or ``alignseq.loc``) must be stored in the configured -[``tool_data_path``](https://github.com/galaxyproject/galaxy/tree/master/tool-data) +[tool_data_path](https://github.com/galaxyproject/galaxy/tree/master/tool-data) directory. In this example, the option names and values are taken from column 0 of the file. @@ -2766,7 +2764,7 @@ wrappers for an example of these. ### Other Ways to Dynamically Generate Options -Though deprecated and discouraged, [``code``](#tool|code) blocks can also be +Though deprecated and discouraged, [code](#tool-code) blocks can also be used to generate dynamic options. ]]> @@ -3583,10 +3581,10 @@ This tag set is contained within the ```` tag set, and it defines the output dataset collection description resulting from the tool's execution. The value of the attribute ``label`` can be acquired from input parameters or metadata in the same way that the command line parameters are (discussed in the -[``command``](#tool|command) directive). +[command](#tool-command) directive). Creating collections in tools is covered in-depth in -[planemo's documentation](http://planemo.readthedocs.io/en/latest/writing_advanced.html#creating-collections). +[planemo's documentation](https://planemo.readthedocs.io/en/latest/writing_advanced.html#creating-collections). ]]> @@ -3704,7 +3702,7 @@ Galaxy, including: * https://github.com/galaxyproject/galaxy/tree/master/test/functional/tools/multi_output_configured.xml More information can be found on Planemo's documentation for -[multiple output files](http://planemo.readthedocs.io/en/latest/writing_advanced.html#multiple-output-files). +[multiple output files](https://planemo.readthedocs.io/en/latest/writing_advanced.html#multiple-output-files). ]]> @@ -3964,7 +3962,7 @@ This directive is contained within an output ``data``'s ``actions`` directive describes modifications to either the output's format or metadata (based on whether ``type`` is ``format`` or ``metadata``). -See [``actions``](#tool|outputs|data|actions) documentation for examples +See [actions](#tool-outputs-data-actions) documentation for examples of this directive. ]]> @@ -4033,7 +4031,7 @@ This directive describes the state of the inputs required to apply an ``action`` (specified as children of the child ``when`` directives to this element) to an output. -See [``actions``](#tool|outputs|data|actions) documentation for examples +See [actions](#tool-outputs-data-actions) documentation for examples of this directive. ]]> @@ -4053,7 +4051,7 @@ conditional logic on. The value of this parameter will be matched against nested @@ -4345,7 +4343,7 @@ response to this directive. order to get the tool's version string. The resulting value will be found in the "Info" field of the history dataset. -Unlike the [``command``](#tool|command) tag, with the exception of the string +Unlike the [command](#tool-command) tag, with the exception of the string ``$__tool_directory__`` this value is taken as a literal and so there is no need to escape values like ``$`` and command inputs are not available for variable substitution. @@ -4369,7 +4367,7 @@ the tool might be: Examples are included in the test tools directory including: - [version_command_plain.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/version_command_plain.xml) -- [version_command_tool_dir.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/version_tool_dir.xml) +- [version_command_tool_dir.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/version_command_tool_dir.xml) - [version_command_interpreter.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/version_command_interpreter.xml) (*deprecated*) ]]> @@ -4599,7 +4597,7 @@ Tools may use exit codes to indicate specific execution errors. Many programs us The exit code's range can be any consecutive group of integers. More advanced ranges, such as noncontiguous ranges, are currently not supported. Ranges can be specified in the form "m:n", where m is the start integer and n is the end integer. If ":n" is specified, then the exit code will be compared against all integers less than or equal to n. If "m:" is used, then the exit code will be compared against all integers greater than or equal to m. If the exit code matches, then the error level is applied and the error's description is added to stderr. If a tool's exit code does not match any of the supplied tags' ranges, then no errors are applied to the tool's execution. -Note that most Unix and Linux variants only support positive integers 0 to 255 for exit codes. If an exit code falls out of the range 0 to 255, the usual convention is to only use the lower 8 bits for the exit code. The only known exception is if a job is broken into subtasks using the tasks runner and one of those tasks is stopped with a POSIX signal. (Note that signals should be used as a last resort for terminating processes.) In those cases, the task will receive -1 times the signal number. For example, suppose that a job uses the tasks runner and 8 tasks are created for the job. If one of the tasks hangs, then a sysadmin may choose to send the "kill" signal, SIGKILL, to the process. In that case, the task (and its job) will exit with an exit code of -9. More on POSIX signals can be found at http://en.wikipedia.org/wiki/Unix_signal as well as man pages on "signal". +Note that most Unix and Linux variants only support positive integers 0 to 255 for exit codes. If an exit code falls out of the range 0 to 255, the usual convention is to only use the lower 8 bits for the exit code. The only known exception is if a job is broken into subtasks using the tasks runner and one of those tasks is stopped with a POSIX signal. (Note that signals should be used as a last resort for terminating processes.) In those cases, the task will receive -1 times the signal number. For example, suppose that a job uses the tasks runner and 8 tasks are created for the job. If one of the tasks hangs, then a sysadmin may choose to send the "kill" signal, SIGKILL, to the process. In that case, the task (and its job) will exit with an exit code of -9. More on POSIX signals can be found at https://en.wikipedia.org/wiki/Unix_signal as well as man pages on "signal". The tag's supported attributes are as follows: @@ -4845,7 +4843,7 @@ well. Likewise, the history menu includes an option allowing users to aggregate all such citations across an analysis in a list of citations. BibTeX entries for citations annotated with DOIs will be fetched by Galaxy from -http://dx.doi.org/ and cached. +https://dx.doi.org/ and cached. ```xml diff --git a/lib/galaxy/util/simplegraph.py b/lib/galaxy/util/simplegraph.py index 88eaed964c3..280eb574086 100644 --- a/lib/galaxy/util/simplegraph.py +++ b/lib/galaxy/util/simplegraph.py @@ -109,7 +109,8 @@ class SimpleGraph( object ): def gen_edge_dicts( self ): """ - Returns a generator that yields node dictionaries in the form: + Returns a generator that yields node dictionaries in the form:: + { 'source': , 'target': , @@ -121,7 +122,8 @@ class SimpleGraph( object ): def as_dict( self ): """ - Returns a dictionary of the form + Returns a dictionary of the form:: + { 'nodes': , 'edges': } """ return { 'nodes': list( self.gen_node_dicts() ), 'edges': list( self.gen_edge_dicts() ) } diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index d49a166decf..d7fc43b7664 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -619,7 +619,7 @@ class HistoryContentsController( BaseAPIController, UsesLibraryMixin, UsesLibrar :returns: archive file for download - .. note: this is a volatile endpoint and settings and behavior may change. + .. note:: this is a volatile endpoint and settings and behavior may change. """ # roughly from: http://stackoverflow.com/a/31976060 (windows, linux) invalid_filename_char_regex = re.compile( r'[:<>|\\\/\?\* "]' ) diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py index 938048f4122..65f083cbcce 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -146,15 +146,15 @@ class Download( object ): def url_download( self, install_dir, downloaded_file_name, download_url, extract=True, checksums={} ): """ - The given download_url can have an extension like #md5#, #sha256#, (or #md5= to support pypi defaults). + The given download_url can have an extension like #md5#, #sha256#, (or #md5= to support pypi defaults). - https://pypi.python.org/packages/source/k/khmer/khmer-1.0.tar.gz#md5#b60639a8b2939836f66495b9a88df757 + https://pypi.python.org/packages/source/k/khmer/khmer-1.0.tar.gz#md5#b60639a8b2939836f66495b9a88df757 - Alternatively, to not break HTTP spec, you can specify md5 and - sha256 as keys in the element. + Alternatively, to not break HTTP spec, you can specify md5 and + sha256 as keys in the element. - This indicates a checksum which will be checked after download. - If the checksum does not match an exception is thrown. + This indicates a checksum which will be checked after download. + If the checksum does not match an exception is thrown. """ file_path = os.path.join( install_dir, downloaded_file_name ) src = None @@ -300,7 +300,8 @@ class AssertDirectoryExists( RecipeStep ): def assert_directory_exists( self, full_path ): """ Return True if a symbolic link or directory exists, but if full_path is a file, - return False. """ + return False. + """ if full_path is None: return False if os.path.isfile( full_path ): @@ -911,20 +912,20 @@ class RegexReplace( RecipeStep ): filtered_actions and dir. This step supports the full range of python's regular expression engine, including backreferences in - the replacement text. + the replacement text. Example:: - Example: - - ^CFLAGS(\s*)=\s*-g\s*-Wall\s*-O2\s*$$ - CFLAGS\1= -g -Wall -O2 -I$$(NCURSES_INCLUDE_PATH)/ncurses/ -I$$(NCURSES_INCLUDE_PATH) -L$$(NCURSES_LIB_PATH) - + + ^CFLAGS(\s*)=\s*-g\s*-Wall\s*-O2\s*$$ + CFLAGS\1= -g -Wall -O2 -I$$(NCURSES_INCLUDE_PATH)/ncurses/ -I$$(NCURSES_INCLUDE_PATH) -L$$(NCURSES_LIB_PATH) + - Before: - CFLAGS = -g -Wall -O2 + Before:: - After: - CFLAGS = -g -Wall -O2 -I$(NCURSES_INCLUDE_PATH)/ncurses/ -I$(NCURSES_INCLUDE_PATH) -L$(NCURSES_LIB_PATH) + CFLAGS = -g -Wall -O2 + After:: + + CFLAGS = -g -Wall -O2 -I$(NCURSES_INCLUDE_PATH)/ncurses/ -I$(NCURSES_INCLUDE_PATH) -L$(NCURSES_LIB_PATH) """ log_file = os.path.join( install_environment.install_dir, basic_util.INSTALLATION_LOG ) if os.path.exists( log_file ): @@ -1011,20 +1012,19 @@ class SetEnvironment( RecipeStep ): The first tag set defines a complex repository dependency like this. This tag set ensures that changeset revision XXX of the repository named package_graphicsmagick_1_3 owned by YYY in the tool shed ZZZ has been - previously installed. + previously installed:: - - - - - ... + + + + + ... - * By the way, there is an env.sh file associated with version 1.3.18 of the graphicsmagick package which looks - something like this (we'll reference this file later in this discussion. - ---- - GRAPHICSMAGICK_ROOT_DIR=//graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick; - export GRAPHICSMAGICK_ROOT_DIR - ---- + By the way, there is an env.sh file associated with version 1.3.18 of the graphicsmagick package which looks + something like this (we'll reference this file later in this discussion):: + + GRAPHICSMAGICK_ROOT_DIR=//graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick; + export GRAPHICSMAGICK_ROOT_DIR The second tag set defines a specific package dependency that has been previously installed (guaranteed by the tag set discussed above) and compiled, where the compiled dependency is needed by the tool dependency currently @@ -1033,44 +1033,42 @@ class SetEnvironment( RecipeStep ): version 2.0.0 of the osra package requires version 1.3.18 of the graphicsmagick package in order to successfully compile. When this tag set is handled, one of the effects is that the env.sh file associated with graphicsmagick version 1.3.18 is "sourced", which undoubtedly sets or alters certain environment variables (e.g. PATH, PYTHONPATH, - etc). + etc):: - - - - - - + + + + + + The third tag set enables discovery of the same required package dependency discussed above for correctly compiling the osra version 2.0.0 package, but in this case the package can be discovered at tool execution time. Using the $ENV[] option as shown in this example, the value of the environment variable named GRAPHICSMAGICK_ROOT_DIR (which was set in the environment using the second tag set described above) will be used to automatically alter the env.sh file associated with the osra version 2.0.0 tool dependency when it is installed into Galaxy. * Refer to where we - discussed the env.sh file for version 1.3.18 of the graphicsmagick package above. + discussed the env.sh file for version 1.3.18 of the graphicsmagick package above:: - - $ENV[GRAPHICSMAGICK_ROOT_DIR]/lib/ - $INSTALL_DIR/potrace/build/lib/ - $INSTALL_DIR/bin - - $INSTALL_DIR/share - + + $ENV[GRAPHICSMAGICK_ROOT_DIR]/lib/ + $INSTALL_DIR/potrace/build/lib/ + $INSTALL_DIR/bin + + $INSTALL_DIR/share + The above tag will produce an env.sh file for version 2.0.0 of the osra package when it it installed into Galaxy that looks something like this. Notice that the path to the gmagick binary is included here since it expands the - defined $ENV[GRAPHICSMAGICK_ROOT_DIR] value in the above tag set. + defined $ENV[GRAPHICSMAGICK_ROOT_DIR] value in the above tag set:: - ---- - LD_LIBRARY_PATH=//graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick/lib/:$LD_LIBRARY_PATH; - export LD_LIBRARY_PATH - LD_LIBRARY_PATH=//osra/1.4.0/YYY/depends_on/XXX/potrace/build/lib/:$LD_LIBRARY_PATH; - export LD_LIBRARY_PATH - PATH=//osra/1.4.0/YYY/depends_on/XXX/bin:$PATH; - export PATH - OSRA_DATA_FILES=//osra/1.4.0/YYY/depends_on/XXX/share; - export OSRA_DATA_FILES - ---- + LD_LIBRARY_PATH=//graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick/lib/:$LD_LIBRARY_PATH; + export LD_LIBRARY_PATH + LD_LIBRARY_PATH=//osra/1.4.0/YYY/depends_on/XXX/potrace/build/lib/:$LD_LIBRARY_PATH; + export LD_LIBRARY_PATH + PATH=//osra/1.4.0/YYY/depends_on/XXX/bin:$PATH; + export PATH + OSRA_DATA_FILES=//osra/1.4.0/YYY/depends_on/XXX/share; + export OSRA_DATA_FILES """ env_var_value = env_var_dict[ 'value' ] # env_var_value is the text of an environment variable tag like this: @@ -1575,7 +1573,6 @@ class SetupPythonEnvironment( Download, RecipeStep ): although it may never be necessary. If initial_download is True, the recipe steps will be filtered and returned and the installation directory (i.e., dir) will be defined and returned. If we're not in the initial download stage, these actions will not occur, and None values will be returned for them. - """ # # diff --git a/scripts/bootstrap_history.py b/scripts/bootstrap_history.py index b2968881765..fdc74493332 100644 --- a/scripts/bootstrap_history.py +++ b/scripts/bootstrap_history.py @@ -46,9 +46,8 @@ DEVTEAM = [ TEMPLATE = """ .. to_doc -------------------------------- %s -------------------------------- +=============================== .. announce_start diff --git a/scripts/bootstrap_history_announce_template.rst b/scripts/bootstrap_history_announce_template.rst deleted file mode 100644 index cf2b51b8f9c..00000000000 --- a/scripts/bootstrap_history_announce_template.rst +++ /dev/null @@ -1,56 +0,0 @@ -=========================================================== -TODO Galaxy Release (v %s) -=========================================================== - -.. include:: _header.rst - -Highlights -=========================================================== - -**Feature1** - Feature description. - -**Feature2** - Feature description. - -**Feature3** - Feature description. - -`Github `__ -=========================================================== - -New - .. code-block:: shell - - % git clone -b master https://github.com/galaxyproject/galaxy.git - -Update to latest stable release - .. code-block:: shell - - % git checkout master && pull --ff-only origin master - -Update to exact version - .. code-block:: shell - - % git checkout v%s - - -`BitBucket `__ -=========================================================== - -Upgrade - .. code-block:: shell - - % hg pull - % hg update latest_%s - - -See `our wiki `__ for additional details regarding the source code locations. - -Release Notes -=========================================================== - -.. include:: %s.rst - :start-after: enhancements - -.. include:: _thanks.rst diff --git a/scripts/galaxy-main b/scripts/galaxy-main index 8a4796d939b..7523c549fcd 100755 --- a/scripts/galaxy-main +++ b/scripts/galaxy-main @@ -57,11 +57,8 @@ REQUIRES_DAEMONIZE_MESSAGE = "Attempted to use Galaxy in daemon mode, but daemon log = logging.getLogger(__name__) -base_file = os.path.basename(__file__) -if base_file == 'main.py': - GALAXY_ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) -elif base_file == 'galaxy-main': - GALAXY_ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) +real_file = os.path.realpath(__file__) +GALAXY_ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(real_file), os.pardir)) GALAXY_LIB_DIR = os.path.join(GALAXY_ROOT_DIR, "lib") DEFAULT_INI_APP = "main" DEFAULT_INIS = ["config/galaxy.ini", "universe_wsgi.ini", "config/galaxy.ini.sample"]