, tag='pre', attrs=[('class', 'screen')]
- uattrs = []
- strattrs = ''
- if attrs:
- for key, value in attrs:
- value = value.replace('>', '>').replace('<', '<').replace('"', '"')
- value = self.bare_ampersand.sub("&", value)
- uattrs.append((key, value))
- strattrs = ''.join([' %s="%s"' % (k, v) for k, v in uattrs])
- if tag in self.elements_no_end_tag:
- self.pieces.append('<%s%s />' % (tag, strattrs))
- else:
- self.pieces.append('<%s%s>' % (tag, strattrs))
-
- def unknown_endtag(self, tag):
- # called for each end tag, e.g. for , tag will be 'pre'
- # Reconstruct the original end tag.
- if tag not in self.elements_no_end_tag:
- self.pieces.append("" % locals())
-
- def handle_charref(self, ref):
- # called for each character reference, e.g. for ' ', ref will be '160'
- # Reconstruct the original character reference.
- if ref.startswith('x'):
- value = unichr(int(ref[1:], 16))
- else:
- value = unichr(int(ref))
-
- if value in _cp1252.keys():
- self.pieces.append('&#%s;' % hex(ord(_cp1252[value]))[1:])
- else:
- self.pieces.append('&#%(ref)s;' % locals())
-
- def handle_entityref(self, ref):
- # called for each entity reference, e.g. for '©', ref will be 'copy'
- # Reconstruct the original entity reference.
- if ref in name2codepoint:
- self.pieces.append('&%(ref)s;' % locals())
- else:
- self.pieces.append('&%(ref)s' % locals())
-
- def handle_data(self, text):
- # called for each block of plain text, i.e. outside of any tag and
- # not containing any character or entity references
- # Store the original text verbatim.
- self.pieces.append(text)
-
- def handle_comment(self, text):
- # called for each HTML comment, e.g.
- # Reconstruct the original comment.
- self.pieces.append('' % locals())
-
- def handle_pi(self, text):
- # called for each processing instruction, e.g.
- # Reconstruct original processing instruction.
- self.pieces.append('' % locals())
-
- def handle_decl(self, text):
- # called for the DOCTYPE, if present, e.g.
- #
- # Reconstruct original DOCTYPE
- self.pieces.append('' % locals())
-
- _new_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9:]*\s*').match
-
- def _scan_name(self, i, declstartpos):
- rawdata = self.rawdata
- n = len(rawdata)
- if i == n:
- return None, -1
- m = self._new_declname_match(rawdata, i)
- if m:
- s = m.group()
- name = s.strip()
- if (i + len(s)) == n:
- return None, -1 # end of buffer
- return name.lower(), m.end()
- else:
- self.handle_data(rawdata)
-# self.updatepos(declstartpos, i)
- return None, -1
-
- def convert_charref(self, name):
- return '&#%s;' % name
-
- def convert_entityref(self, name):
- return '&%s;' % name
-
- def output(self):
- '''Return processed HTML as a single string'''
- return ''.join(self.pieces)
-
-
-class _HTMLSanitizer(_BaseHTMLProcessor):
- acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', 'article',
- 'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', 'canvas',
- 'caption', 'center', 'cite', 'code', 'col', 'colgroup', 'command',
- 'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', 'dialog', 'dir',
- 'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', 'figure', 'footer',
- 'font', 'form', 'header', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i',
- 'img', 'input', 'ins', 'keygen', 'kbd', 'label', 'legend', 'li', 'm', 'map',
- 'menu', 'meter', 'multicol', 'nav', 'nextid', 'ol', 'output', 'optgroup',
- 'option', 'p', 'pre', 'progress', 'q', 's', 'samp', 'section', 'select',
- 'small', 'sound', 'source', 'spacer', 'span', 'strike', 'strong', 'sub',
- 'sup', 'table', 'tbody', 'td', 'textarea', 'time', 'tfoot', 'th', 'thead',
- 'tr', 'tt', 'u', 'ul', 'var', 'video', 'noscript']
-
- acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey',
- 'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis',
- 'background', 'balance', 'bgcolor', 'bgproperties', 'border',
- 'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding',
- 'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff',
- 'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color', 'cols',
- 'colspan', 'compact', 'contenteditable', 'controls', 'coords', 'data',
- 'datafld', 'datapagesize', 'datasrc', 'datetime', 'default', 'delay',
- 'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end', 'face', 'for',
- 'form', 'frame', 'galleryimg', 'gutter', 'headers', 'height', 'hidefocus',
- 'hidden', 'high', 'href', 'hreflang', 'hspace', 'icon', 'id', 'inputmode',
- 'ismap', 'keytype', 'label', 'leftspacing', 'lang', 'list', 'longdesc',
- 'loop', 'loopcount', 'loopend', 'loopstart', 'low', 'lowsrc', 'max',
- 'maxlength', 'media', 'method', 'min', 'multiple', 'name', 'nohref',
- 'noshade', 'nowrap', 'open', 'optimum', 'pattern', 'ping', 'point-size',
- 'prompt', 'pqg', 'radiogroup', 'readonly', 'rel', 'repeat-max',
- 'repeat-min', 'replace', 'required', 'rev', 'rightspacing', 'rows',
- 'rowspan', 'rules', 'scope', 'selected', 'shape', 'size', 'span', 'src',
- 'start', 'step', 'summary', 'suppress', 'tabindex', 'target', 'template',
- 'title', 'toppadding', 'type', 'unselectable', 'usemap', 'urn', 'valign',
- 'value', 'variable', 'volume', 'vspace', 'vrml', 'width', 'wrap',
- 'xml:lang']
-
- unacceptable_elements_with_end_tag = ['script', 'applet', 'style']
-
- acceptable_css_properties = ['azimuth', 'background-color',
- 'border-bottom-color', 'border-collapse', 'border-color',
- 'border-left-color', 'border-right-color', 'border-top-color', 'clear',
- 'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font',
- 'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight',
- 'height', 'letter-spacing', 'line-height', 'overflow', 'pause',
- 'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness',
- 'speak', 'speak-header', 'speak-numeral', 'speak-punctuation',
- 'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent',
- 'unicode-bidi', 'vertical-align', 'voice-family', 'volume',
- 'white-space', 'width']
-
- # survey of common keywords found in feeds
- acceptable_css_keywords = ['auto', 'aqua', 'black', 'block', 'blue',
- 'bold', 'both', 'bottom', 'brown', 'center', 'collapse', 'dashed',
- 'dotted', 'fuchsia', 'gray', 'green', '!important', 'italic', 'left',
- 'lime', 'maroon', 'medium', 'none', 'navy', 'normal', 'nowrap', 'olive',
- 'pointer', 'purple', 'red', 'right', 'solid', 'silver', 'teal', 'top',
- 'transparent', 'underline', 'white', 'yellow']
-
- valid_css_values = re.compile('^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|' +
- '\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$')
-
- mathml_elements = ['annotation', 'annotation-xml', 'maction', 'math',
- 'merror', 'mfenced', 'mfrac', 'mi', 'mmultiscripts', 'mn', 'mo', 'mover', 'mpadded',
- 'mphantom', 'mprescripts', 'mroot', 'mrow', 'mspace', 'msqrt', 'mstyle',
- 'msub', 'msubsup', 'msup', 'mtable', 'mtd', 'mtext', 'mtr', 'munder',
- 'munderover', 'none', 'semantics']
-
- mathml_attributes = ['actiontype', 'align', 'columnalign', 'columnalign',
- 'columnalign', 'close', 'columnlines', 'columnspacing', 'columnspan', 'depth',
- 'display', 'displaystyle', 'encoding', 'equalcolumns', 'equalrows',
- 'fence', 'fontstyle', 'fontweight', 'frame', 'height', 'linethickness',
- 'lspace', 'mathbackground', 'mathcolor', 'mathvariant', 'mathvariant',
- 'maxsize', 'minsize', 'open', 'other', 'rowalign', 'rowalign', 'rowalign',
- 'rowlines', 'rowspacing', 'rowspan', 'rspace', 'scriptlevel', 'selection',
- 'separator', 'separators', 'stretchy', 'width', 'width', 'xlink:href',
- 'xlink:show', 'xlink:type', 'xmlns', 'xmlns:xlink']
-
- # svgtiny - foreignObject + linearGradient + radialGradient + stop
- svg_elements = ['a', 'animate', 'animateColor', 'animateMotion',
- 'animateTransform', 'circle', 'defs', 'desc', 'ellipse', 'foreignObject',
- 'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern',
- 'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph', 'mpath',
- 'path', 'polygon', 'polyline', 'radialGradient', 'rect', 'set', 'stop',
- 'svg', 'switch', 'text', 'title', 'tspan', 'use']
-
- # svgtiny + class + opacity + offset + xmlns + xmlns:xlink
- svg_attributes = ['accent-height', 'accumulate', 'additive', 'alphabetic',
- 'arabic-form', 'ascent', 'attributeName', 'attributeType',
- 'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height',
- 'class', 'color', 'color-rendering', 'content', 'cx', 'cy', 'd', 'dx',
- 'dy', 'descent', 'display', 'dur', 'end', 'fill', 'fill-opacity',
- 'fill-rule', 'font-family', 'font-size', 'font-stretch', 'font-style',
- 'font-variant', 'font-weight', 'from', 'fx', 'fy', 'g1', 'g2',
- 'glyph-name', 'gradientUnits', 'hanging', 'height', 'horiz-adv-x',
- 'horiz-origin-x', 'id', 'ideographic', 'k', 'keyPoints', 'keySplines',
- 'keyTimes', 'lang', 'mathematical', 'marker-end', 'marker-mid',
- 'marker-start', 'markerHeight', 'markerUnits', 'markerWidth', 'max',
- 'min', 'name', 'offset', 'opacity', 'orient', 'origin',
- 'overline-position', 'overline-thickness', 'panose-1', 'path',
- 'pathLength', 'points', 'preserveAspectRatio', 'r', 'refX', 'refY',
- 'repeatCount', 'repeatDur', 'requiredExtensions', 'requiredFeatures',
- 'restart', 'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv',
- 'stop-color', 'stop-opacity', 'strikethrough-position',
- 'strikethrough-thickness', 'stroke', 'stroke-dasharray',
- 'stroke-dashoffset', 'stroke-linecap', 'stroke-linejoin',
- 'stroke-miterlimit', 'stroke-opacity', 'stroke-width', 'systemLanguage',
- 'target', 'text-anchor', 'to', 'transform', 'type', 'u1', 'u2',
- 'underline-position', 'underline-thickness', 'unicode', 'unicode-range',
- 'units-per-em', 'values', 'version', 'viewBox', 'visibility', 'width',
- 'widths', 'x', 'x-height', 'x1', 'x2', 'xlink:actuate', 'xlink:arcrole',
- 'xlink:href', 'xlink:role', 'xlink:show', 'xlink:title', 'xlink:type',
- 'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y', 'y1',
- 'y2', 'zoomAndPan']
-
- svg_attr_map = None
- svg_elem_map = None
-
- acceptable_svg_properties = ['fill', 'fill-opacity', 'fill-rule',
- 'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin',
- 'stroke-opacity']
-
- def reset(self):
- _BaseHTMLProcessor.reset(self)
- self.unacceptablestack = 0
- self.mathmlOK = 0
- self.svgOK = 0
-
- def unknown_starttag(self, tag, attrs):
- acceptable_attributes = self.acceptable_attributes
- keymap = {}
- if tag not in self.acceptable_elements or self.svgOK:
- if tag in self.unacceptable_elements_with_end_tag:
- self.unacceptablestack += 1
-
- # not otherwise acceptable, perhaps it is MathML or SVG?
- if tag == 'math' and ('xmlns', 'http://www.w3.org/1998/Math/MathML') in attrs:
- self.mathmlOK += 1
- if tag == 'svg' and ('xmlns', 'http://www.w3.org/2000/svg') in attrs:
- self.svgOK += 1
-
- # chose acceptable attributes based on tag class, else bail
- if self.mathmlOK and tag in self.mathml_elements:
- acceptable_attributes = self.mathml_attributes
- elif self.svgOK and tag in self.svg_elements:
- # for most vocabularies, lowercasing is a good idea. Many
- # svg elements, however, are camel case
- if not self.svg_attr_map:
- lower = [attr.lower() for attr in self.svg_attributes]
- mix = [a for a in self.svg_attributes if a not in lower]
- self.svg_attributes = lower
- self.svg_attr_map = dict([(a.lower(), a) for a in mix])
-
- lower = [attr.lower() for attr in self.svg_elements]
- mix = [a for a in self.svg_elements if a not in lower]
- self.svg_elements = lower
- self.svg_elem_map = dict([(a.lower(), a) for a in mix])
- acceptable_attributes = self.svg_attributes
- tag = self.svg_elem_map.get(tag, tag)
- keymap = self.svg_attr_map
- elif tag not in self.acceptable_elements:
- return
-
- # declare xlink namespace, if needed
- if self.mathmlOK or self.svgOK:
- if any(map(lambda n, v: n.startswith('xlink:'), attrs)):
- if not ('xmlns:xlink', 'http://www.w3.org/1999/xlink') in attrs:
- attrs.append(('xmlns:xlink', 'http://www.w3.org/1999/xlink'))
-
- clean_attrs = []
- for key, value in self.normalize_attrs(attrs):
- if key == "href" and value.strip().startswith("javascript"):
- pass
- elif key in acceptable_attributes:
- key = keymap.get(key, key)
- clean_attrs.append((key, value))
- elif key == 'style':
- pass
- # clean_value = self.sanitize_style(value)
- # if clean_value: clean_attrs.append((key,clean_value))
- _BaseHTMLProcessor.unknown_starttag(self, tag, clean_attrs)
-
- def unknown_endtag(self, tag):
- if tag not in self.acceptable_elements:
- if tag in self.unacceptable_elements_with_end_tag:
- self.unacceptablestack -= 1
- if self.mathmlOK and tag in self.mathml_elements:
- if tag == 'math' and self.mathmlOK:
- self.mathmlOK -= 1
- elif self.svgOK and tag in self.svg_elements:
- tag = self.svg_elem_map.get(tag, tag)
- if tag == 'svg' and self.svgOK:
- self.svgOK -= 1
- else:
- return
- _BaseHTMLProcessor.unknown_endtag(self, tag)
-
- def handle_pi(self, text):
- pass
-
- def handle_decl(self, text):
- pass
-
- def handle_data(self, text):
- if not self.unacceptablestack:
- _BaseHTMLProcessor.handle_data(self, text)
-
- def sanitize_style(self, style):
- # disallow urls
- style = re.compile('url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ', style)
-
- # gauntlet
- if not re.match("""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style):
- return ''
- if not re.match("^(\s*[-\w]+\s*:\s*[^:;]*(;|$))*$", style):
- return ''
-
- clean = []
- for prop, value in re.findall("([-\w]+)\s*:\s*([^:;]*)", style):
- if not value:
- continue
- if prop.lower() in self.acceptable_css_properties:
- clean.append(prop + ': ' + value + ';')
- elif prop.split('-')[0].lower() in ['background', 'border', 'margin', 'padding']:
- for keyword in value.split():
- if keyword not in self.acceptable_css_keywords and \
- not self.valid_css_values.match(keyword):
- break
- else:
- clean.append(prop + ': ' + value + ';')
- elif self.svgOK and prop.lower() in self.acceptable_svg_properties:
- clean.append(prop + ': ' + value + ';')
- return ' '.join(clean)
-
-
-def sanitize_html(htmlSource, encoding="utf-8", type="text/html"):
- p = _HTMLSanitizer(encoding, type)
- p.feed(unicodify(htmlSource, encoding))
- data = p.output()
- data = data.strip().replace('\r\n', '\n')
- return data
+def sanitize_html(htmlSource):
+ return bleach.clean(htmlSource, tags=_acceptable_elements, attributes=_acceptable_attributes, strip=True)
diff --git a/lib/galaxy/util/validation.py b/lib/galaxy/util/validation.py
index a851148c5e5..1ecf60d129e 100644
--- a/lib/galaxy/util/validation.py
+++ b/lib/galaxy/util/validation.py
@@ -12,13 +12,13 @@ def validate_and_sanitize_basestring(key, val):
if not isinstance(val, string_types):
raise exceptions.RequestParameterInvalidException('%s must be a string or unicode: %s'
% (key, str(type(val))))
- return sanitize_html(val, 'utf-8', 'text/html')
+ return sanitize_html(val)
def validate_and_sanitize_basestring_list(key, val):
try:
assert isinstance(val, list)
- return [sanitize_html(t, 'utf-8', 'text/html') for t in val]
+ return [sanitize_html(t) for t in val]
except (AssertionError, TypeError):
raise exceptions.RequestParameterInvalidException('%s must be a list of strings: %s'
% (key, str(type(val))))
@@ -32,7 +32,7 @@ def validate_boolean(key, val):
# TODO:
-# def validate_integer( self, key, val, min, max ):
-# def validate_float( self, key, val, min, max ):
-# def validate_number( self, key, val, min, max ):
-# def validate_genome_build( self, key, val ):
+# def validate_integer(self, key, val, min, max):
+# def validate_float(self, key, val, min, max):
+# def validate_number(self, key, val, min, max):
+# def validate_genome_build(self, key, val):
diff --git a/lib/galaxy/visualization/plugins/resource_parser.py b/lib/galaxy/visualization/plugins/resource_parser.py
index 1ac6b65464a..ebc1754a277 100644
--- a/lib/galaxy/visualization/plugins/resource_parser.py
+++ b/lib/galaxy/visualization/plugins/resource_parser.py
@@ -29,7 +29,7 @@ class ResourceParser(object):
new keys (e.g. dataset_id="NNN" -> hda=,
+ tag='pre', attrs=[('class', 'screen')]
+ """
# If ignoring content, just increment tag count and ignore.
if self.ignore_content:
@@ -264,17 +357,25 @@ class _PageContentProcessor(_BaseHTMLProcessor):
return
# Default behavior: not ignoring and no embedded content.
- _BaseHTMLProcessor.unknown_starttag(self, tag, attrs)
+ uattrs = []
+ strattrs = ''
+ if attrs:
+ for key, value in attrs:
+ value = value.replace('>', '>').replace('<', '<').replace('"', '"')
+ value = self.bare_ampersand.sub("&", value)
+ uattrs.append((key, value))
+ strattrs = ''.join(' %s="%s"' % (k, v) for k, v in uattrs)
+ if tag in self.elements_no_end_tag:
+ self.pieces.append('<%s%s />' % (tag, strattrs))
+ else:
+ self.pieces.append('<%s%s>' % (tag, strattrs))
- def handle_data(self, text):
- """ Called for each block of plain text. """
- if self.ignore_content:
- return
- _BaseHTMLProcessor.handle_data(self, text)
-
- def unknown_endtag(self, tag):
- """ Called for each end tag. """
+ def handle_endtag(self, tag):
+ """
+ Called for each end tag
+ E.g. for , tag will be 'pre'
+ """
# If ignoring content, see if current tag is the end of content to ignore.
if self.ignore_content:
self.num_open_tags_for_ignore -= 1
@@ -283,8 +384,63 @@ class _PageContentProcessor(_BaseHTMLProcessor):
self.ignore_content = False
return
- # Default behavior:
- _BaseHTMLProcessor.unknown_endtag(self, tag)
+ # Default behavior: reconstruct the original end tag.
+ if tag not in self.elements_no_end_tag:
+ self.pieces.append("" % tag)
+
+ def handle_charref(self, ref):
+ # called for each character reference, e.g. for ' ', ref will be '160'
+ # Reconstruct the original character reference.
+ ref = ref.lower()
+ if ref.startswith('x'):
+ value = int(ref[1:], 16)
+ else:
+ value = int(ref)
+
+ if value in _cp1252:
+ self.pieces.append('&#%s;' % hex(ord(_cp1252[value]))[1:])
+ else:
+ self.pieces.append('&#%s;' % ref)
+
+ def handle_entityref(self, ref):
+ # called for each entity reference, e.g. for '©', ref will be 'copy'
+ # Reconstruct the original entity reference.
+ if ref in name2codepoint or ref == 'apos':
+ self.pieces.append('&%s;' % ref)
+ else:
+ self.pieces.append('&%s' % ref)
+
+ def handle_data(self, text):
+ """
+ Called for each block of plain text
+
+ Called outside of any tag and not containing any character or entity
+ references. Store the original text verbatim.
+ """
+ if self.ignore_content:
+ return
+ self.pieces.append(text)
+
+ def handle_comment(self, text):
+ # called for each HTML comment, e.g.
+ # Reconstruct the original comment.
+ self.pieces.append('' % text)
+
+ def handle_decl(self, text):
+ # called for the DOCTYPE, if present, e.g.
+ #
+ # Reconstruct original DOCTYPE
+ self.pieces.append('' % text)
+
+ def handle_pi(self, text):
+ # called for each processing instruction, e.g.
+ # Reconstruct original processing instruction.
+ self.pieces.append('' % text)
+
+ def output(self):
+ '''Return processed HTML as a single string'''
+ return ''.join(self.pieces)
class PageController(BaseUIController, SharableMixin,
@@ -387,7 +543,7 @@ class PageController(BaseUIController, SharableMixin,
p.slug = p_slug
p.user = user
if p_annotation:
- p_annotation = sanitize_html(p_annotation, 'utf-8', 'text/html')
+ p_annotation = sanitize_html(p_annotation)
self.add_item_annotation(trans.sa_session, user, p, p_annotation)
# And the first (empty) page revision
p_revision = model.PageRevision()
@@ -449,7 +605,7 @@ class PageController(BaseUIController, SharableMixin,
p.title = p_title
p.slug = p_slug
if p_annotation:
- p_annotation = sanitize_html(p_annotation, 'utf-8', 'text/html')
+ p_annotation = sanitize_html(p_annotation)
self.add_item_annotation(trans.sa_session, user, p, p_annotation)
trans.sa_session.add(p)
trans.sa_session.flush()
@@ -550,7 +706,7 @@ class PageController(BaseUIController, SharableMixin,
assert page.user == trans.user
# Sanitize content
- content = sanitize_html(content, 'utf-8', 'text/html')
+ content = sanitize_html(content)
# Add a new revision to the page with the provided content.
page_revision = model.PageRevision()
@@ -567,7 +723,7 @@ class PageController(BaseUIController, SharableMixin,
item = trans.sa_session.query(item_class).filter_by(id=item_id).first()
if not item:
raise RuntimeError("cannot find annotated item")
- text = sanitize_html(annotation_dict['text'], 'utf-8', 'text/html')
+ text = sanitize_html(annotation_dict['text'])
# Add/update annotation.
if item_id and item_class and text:
@@ -615,8 +771,10 @@ class PageController(BaseUIController, SharableMixin,
self.security_check(trans, page, False, True)
# Process page content.
- processor = _PageContentProcessor(trans, 'utf-8', 'text/html', self._get_embed_html)
+ processor = _PageContentProcessor(trans, self._get_embed_html)
processor.feed(page.latest_revision.content)
+ # Output is string, so convert to unicode for display.
+ page_content = unicodify(processor.output(), 'utf-8')
# Get rating data.
user_item_rating = 0
@@ -628,8 +786,6 @@ class PageController(BaseUIController, SharableMixin,
user_item_rating = 0
ave_item_rating, num_ratings = self.get_ave_item_rating_data(trans.sa_session, page)
- # Output is string, so convert to unicode for display.
- page_content = unicodify(processor.output(), 'utf-8')
return trans.fill_template_mako("page/display.mako", item=page,
item_data=page_content,
user_item_rating=user_item_rating,
diff --git a/lib/galaxy/webapps/galaxy/controllers/visualization.py b/lib/galaxy/webapps/galaxy/controllers/visualization.py
index 0232494646e..4d618b36057 100644
--- a/lib/galaxy/webapps/galaxy/controllers/visualization.py
+++ b/lib/galaxy/webapps/galaxy/controllers/visualization.py
@@ -620,7 +620,7 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization
v.slug = v_slug
v.dbkey = v_dbkey
if v_annotation:
- v_annotation = sanitize_html(v_annotation, 'utf-8', 'text/html')
+ v_annotation = sanitize_html(v_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), v, v_annotation)
trans.sa_session.add(v)
trans.sa_session.flush()
diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py
index 06b75087eed..8aac37e9638 100644
--- a/lib/galaxy/webapps/galaxy/controllers/workflow.py
+++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py
@@ -4,10 +4,10 @@ import base64
import json
import logging
import os
-import sgmllib
import requests
from markupsafe import escape
+from six.moves.html_parser import HTMLParser
from six.moves.http_client import HTTPConnection
from sqlalchemy import and_
from sqlalchemy.orm import eagerload, joinedload, lazyload, undefer
@@ -153,16 +153,17 @@ class StoredWorkflowAllPublishedGrid(grids.Grid):
self.model_class.deleted == expression.false())
-# Simple SGML parser to get all content in a single tag.
-class SingleTagContentsParser(sgmllib.SGMLParser):
+# Simple HTML parser to get all content in a single tag.
+class SingleTagContentsParser(HTMLParser):
def __init__(self, target_tag):
- sgmllib.SGMLParser.__init__(self)
+ # Cannot use super() because HTMLParser is an old-style class in Python2
+ HTMLParser.__init__(self)
self.target_tag = target_tag
self.cur_tag = None
self.tag_content = ""
- def unknown_starttag(self, tag, attrs):
+ def handle_starttag(self, tag, attrs):
""" Called for each start tag. """
self.cur_tag = tag
@@ -404,7 +405,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
stored = self.get_stored_workflow(trans, id)
if new_annotation:
# Sanitize annotation before adding it.
- new_annotation = sanitize_html(new_annotation, 'utf-8', 'text/html')
+ new_annotation = sanitize_html(new_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), stored, new_annotation)
trans.sa_session.flush()
return new_annotation
@@ -547,7 +548,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
workflow.stored_workflow = stored_workflow
stored_workflow.latest_workflow = workflow
# Add annotation.
- workflow_annotation = sanitize_html(workflow_annotation, 'utf-8', 'text/html')
+ workflow_annotation = sanitize_html(workflow_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), stored_workflow, workflow_annotation)
# Persist
session = trans.sa_session
@@ -573,7 +574,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
workflow.stored_workflow = stored_workflow
stored_workflow.latest_workflow = workflow
# Add annotation.
- workflow_annotation = sanitize_html(workflow_annotation, 'utf-8', 'text/html')
+ workflow_annotation = sanitize_html(workflow_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), stored_workflow, workflow_annotation)
# Persist
diff --git a/lib/tool_shed/util/workflow_util.py b/lib/tool_shed/util/workflow_util.py
index 38a7edd8f3e..0604759c040 100644
--- a/lib/tool_shed/util/workflow_util.py
+++ b/lib/tool_shed/util/workflow_util.py
@@ -287,7 +287,7 @@ def get_workflow_from_dict(trans, workflow_dict, tools_metadata, repository_id,
if trans.webapp.name == 'galaxy':
annotation = step_dict.get('annotation', '')
if annotation:
- annotation = sanitize_html(annotation, 'utf-8', 'text/html')
+ annotation = sanitize_html(annotation)
new_step_annotation = trans.model.WorkflowStepAnnotationAssociation()
new_step_annotation.annotation = annotation
new_step_annotation.user = trans.user
@@ -398,7 +398,7 @@ def save_workflow(trans, workflow, workflow_dict=None):
stored.latest_workflow = workflow
stored.user = trans.user
if workflow_dict and workflow_dict.get('annotation', ''):
- annotation = sanitize_html(workflow_dict['annotation'], 'utf-8', 'text/html')
+ annotation = sanitize_html(workflow_dict['annotation'])
new_annotation = trans.model.StoredWorkflowAnnotationAssociation()
new_annotation.annotation = annotation
new_annotation.user = trans.user
diff --git a/scripts/microbes/BeautifulSoup.py b/scripts/microbes/BeautifulSoup.py
deleted file mode 100644
index a49381ae217..00000000000
--- a/scripts/microbes/BeautifulSoup.py
+++ /dev/null
@@ -1,1804 +0,0 @@
-"""Beautiful Soup
-Elixir and Tonic
-"The Screen-Scraper's Friend"
-https://www.crummy.com/software/BeautifulSoup/
-
-Beautiful Soup parses a (possibly invalid) XML or HTML document into a
-tree representation. It provides methods and Pythonic idioms that make
-it easy to navigate, search, and modify the tree.
-
-A well-structured XML/HTML document yields a well-behaved data
-structure. An ill-structured XML/HTML document yields a
-correspondingly ill-behaved data structure. If your document is only
-locally well-structured, you can use this library to find and process
-the well-structured part of it.
-
-Beautiful Soup works with Python 2.2 and up. It has no external
-dependencies, but you'll have more success at converting data to UTF-8
-if you also install these three packages:
-
-* chardet, for auto-detecting character encodings
- http://chardet.feedparser.org/
-* cjkcodecs and iconv_codec, which add more encodings to the ones supported
- by stock Python.
- http://cjkpython.i18n.org/
-
-Beautiful Soup defines classes for two main parsing strategies:
-
- * BeautifulStoneSoup, for parsing XML, SGML, or your domain-specific
- language that kind of looks like XML.
-
- * BeautifulSoup, for parsing run-of-the-mill HTML code, be it valid
- or invalid. This class has web browser-like heuristics for
- obtaining a sensible parse tree in the face of common HTML errors.
-
-Beautiful Soup also defines a class (UnicodeDammit) for autodetecting
-the encoding of an HTML or XML document, and converting it to
-Unicode. Much of this code is taken from Mark Pilgrim's Universal Feed
-Parser.
-
-For more than you ever wanted to know about Beautiful Soup, see the
-documentation:
-http://www.crummy.com/software/BeautifulSoup/documentation.html
-"""
-from __future__ import (
- generators,
- print_function
-)
-
-import codecs
-import re
-import sgmllib
-import string
-import sys
-from sgmllib import (
- SGMLParseError,
- SGMLParser
-)
-
-import six
-from six.moves.html_entities import name2codepoint
-
-__author__ = "Leonard Richardson (crummy.com)"
-__contributors__ = ["Sam Ruby (intertwingly.net)",
- "the unwitting Mark Pilgrim (diveintomark.org)",
- "http://www.crummy.com/software/BeautifulSoup/AUTHORS.html"]
-__version__ = "3.0.3"
-__copyright__ = "Copyright (c) 2004-2006 Leonard Richardson"
-__license__ = "PSF"
-
-# This RE makes Beautiful Soup able to parse XML with namespaces.
-sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*')
-
-# This RE makes Beautiful Soup capable of recognizing numeric character
-# references that use hexadecimal.
-sgmllib.charref = re.compile('&#(\d+|x[0-9a-fA-F]+);')
-
-DEFAULT_OUTPUT_ENCODING = "utf-8"
-
-
-# First, the classes that represent markup elements.
-class PageElement:
- """Contains the navigational information for some part of the page
- (either a tag or a piece of text)"""
-
- def setup(self, parent=None, previous=None):
- """Sets up the initial relations between this element and
- other elements."""
- self.parent = parent
- self.previous = previous
- self.next = None
- self.previousSibling = None
- self.nextSibling = None
- if self.parent and self.parent.contents:
- self.previousSibling = self.parent.contents[-1]
- self.previousSibling.nextSibling = self
-
- def replaceWith(self, replaceWith):
- oldParent = self.parent
- myIndex = self.parent.contents.index(self)
- if hasattr(replaceWith, 'parent') and replaceWith.parent == self.parent:
- # We're replacing this element with one of its siblings.
- index = self.parent.contents.index(replaceWith)
- if index and index < myIndex:
- # Furthermore, it comes before this element. That
- # means that when we extract it, the index of this
- # element will change.
- myIndex = myIndex - 1
- self.extract()
- oldParent.insert(myIndex, replaceWith)
-
- def extract(self):
- """Destructively rips this element out of the tree."""
- if self.parent:
- try:
- self.parent.contents.remove(self)
- except ValueError:
- pass
-
- # Find the two elements that would be next to each other if
- # this element (and any children) hadn't been parsed. Connect
- # the two.
- lastChild = self._lastRecursiveChild()
- nextElement = lastChild.next
-
- if self.previous:
- self.previous.next = nextElement
- if nextElement:
- nextElement.previous = self.previous
- self.previous = None
- lastChild.next = None
-
- self.parent = None
- if self.previousSibling:
- self.previousSibling.nextSibling = self.nextSibling
- if self.nextSibling:
- self.nextSibling.previousSibling = self.previousSibling
- self.previousSibling = self.nextSibling = None
-
- def _lastRecursiveChild(self):
- "Finds the last element beneath this object to be parsed."
- lastChild = self
- while hasattr(lastChild, 'contents') and lastChild.contents:
- lastChild = lastChild.contents[-1]
- return lastChild
-
- def insert(self, position, newChild):
- if isinstance(newChild, six.string_types) and \
- not isinstance(newChild, NavigableString):
- newChild = NavigableString(newChild)
-
- position = min(position, len(self.contents))
- if hasattr(newChild, 'parent') and newChild.parent is not None:
- # We're 'inserting' an element that's already one
- # of this object's children.
- if newChild.parent == self:
- index = self.find(newChild)
- if index and index < position:
- # Furthermore we're moving it further down the
- # list of this object's children. That means that
- # when we extract this element, our target index
- # will jump down one.
- position = position - 1
- newChild.extract()
-
- newChild.parent = self
- previousChild = None
- if position == 0:
- newChild.previousSibling = None
- newChild.previous = self
- else:
- previousChild = self.contents[position - 1]
- newChild.previousSibling = previousChild
- newChild.previousSibling.nextSibling = newChild
- newChild.previous = previousChild._lastRecursiveChild()
- if newChild.previous:
- newChild.previous.next = newChild
-
- newChildsLastElement = newChild._lastRecursiveChild()
-
- if position >= len(self.contents):
- newChild.nextSibling = None
-
- parent = self
- parentsNextSibling = None
- while not parentsNextSibling:
- parentsNextSibling = parent.nextSibling
- parent = parent.parent
- if not parent: # This is the last element in the document.
- break
- if parentsNextSibling:
- newChildsLastElement.next = parentsNextSibling
- else:
- newChildsLastElement.next = None
- else:
- nextChild = self.contents[position]
- newChild.nextSibling = nextChild
- if newChild.nextSibling:
- newChild.nextSibling.previousSibling = newChild
- newChildsLastElement.next = nextChild
-
- if newChildsLastElement.next:
- newChildsLastElement.next.previous = newChildsLastElement
- self.contents.insert(position, newChild)
-
- def findNext(self, name=None, attrs={}, text=None, **kwargs):
- """Returns the first item that matches the given criteria and
- appears after this Tag in the document."""
- return self._findOne(self.findAllNext, name, attrs, text, **kwargs)
-
- def findAllNext(self, name=None, attrs={}, text=None, limit=None,
- **kwargs):
- """Returns all items that match the given criteria and appear
- before after Tag in the document."""
- return self._findAll(name, attrs, text, limit, self.nextGenerator)
-
- def findNextSibling(self, name=None, attrs={}, text=None, **kwargs):
- """Returns the closest sibling to this Tag that matches the
- given criteria and appears after this Tag in the document."""
- return self._findOne(self.findNextSiblings, name, attrs, text,
- **kwargs)
-
- def findNextSiblings(self, name=None, attrs={}, text=None, limit=None,
- **kwargs):
- """Returns the siblings of this Tag that match the given
- criteria and appear after this Tag in the document."""
- return self._findAll(name, attrs, text, limit,
- self.nextSiblingGenerator, **kwargs)
- fetchNextSiblings = findNextSiblings # Compatibility with pre-3.x
-
- def findPrevious(self, name=None, attrs={}, text=None, **kwargs):
- """Returns the first item that matches the given criteria and
- appears before this Tag in the document."""
- return self._findOne(self.findAllPrevious, name, attrs, text, **kwargs)
-
- def findAllPrevious(self, name=None, attrs={}, text=None, limit=None,
- **kwargs):
- """Returns all items that match the given criteria and appear
- before this Tag in the document."""
- return self._findAll(name, attrs, text, limit, self.previousGenerator,
- **kwargs)
- fetchPrevious = findAllPrevious # Compatibility with pre-3.x
-
- def findPreviousSibling(self, name=None, attrs={}, text=None, **kwargs):
- """Returns the closest sibling to this Tag that matches the
- given criteria and appears before this Tag in the document."""
- return self._findOne(self.findPreviousSiblings, name, attrs, text,
- **kwargs)
-
- def findPreviousSiblings(self, name=None, attrs={}, text=None,
- limit=None, **kwargs):
- """Returns the siblings of this Tag that match the given
- criteria and appear before this Tag in the document."""
- return self._findAll(name, attrs, text, limit,
- self.previousSiblingGenerator, **kwargs)
- fetchPreviousSiblings = findPreviousSiblings # Compatibility with pre-3.x
-
- def findParent(self, name=None, attrs={}, **kwargs):
- """Returns the closest parent of this Tag that matches the given
- criteria."""
- # NOTE: We can't use _findOne because findParents takes a different
- # set of arguments.
- r = None
- l = self.findParents(name, attrs, 1)
- if l:
- r = l[0]
- return r
-
- def findParents(self, name=None, attrs={}, limit=None, **kwargs):
- """Returns the parents of this Tag that match the given
- criteria."""
-
- return self._findAll(name, attrs, None, limit, self.parentGenerator,
- **kwargs)
- fetchParents = findParents # Compatibility with pre-3.x
-
- # These methods do the real heavy lifting.
-
- def _findOne(self, method, name, attrs, text, **kwargs):
- r = None
- l = method(name, attrs, text, 1, **kwargs)
- if l:
- r = l[0]
- return r
-
- def _findAll(self, name, attrs, text, limit, generator, **kwargs):
- "Iterates over a generator looking for things that match."
-
- if isinstance(name, SoupStrainer):
- strainer = name
- else:
- # Build a SoupStrainer
- strainer = SoupStrainer(name, attrs, text, **kwargs)
- results = ResultSet(strainer)
- g = generator()
- while True:
- try:
- i = next(g)
- except StopIteration:
- break
- if i:
- found = strainer.search(i)
- if found:
- results.append(found)
- if limit and len(results) >= limit:
- break
- return results
-
- # These Generators can be used to navigate starting from both
- # NavigableStrings and Tags.
- def nextGenerator(self):
- i = self
- while i:
- i = i.next
- yield i
-
- def nextSiblingGenerator(self):
- i = self
- while i:
- i = i.nextSibling
- yield i
-
- def previousGenerator(self):
- i = self
- while i:
- i = i.previous
- yield i
-
- def previousSiblingGenerator(self):
- i = self
- while i:
- i = i.previousSibling
- yield i
-
- def parentGenerator(self):
- i = self
- while i:
- i = i.parent
- yield i
-
- # Utility methods
- def substituteEncoding(self, str, encoding=None):
- encoding = encoding or "utf-8"
- return str.replace("%SOUP-ENCODING%", encoding)
-
- def toEncoding(self, s, encoding=None):
- """Encodes an object to a string in some encoding, or to Unicode.
- ."""
- if not isinstance(s, six.string_types) and not isinstance(s, six.binary_type):
- s = str(s)
- if isinstance(s, six.text_type):
- if encoding:
- s = s.encode(encoding)
- else:
- if encoding:
- s = s.encode(encoding)
- else:
- s = six.text_type(s)
- return s
-
-
-class NavigableString(six.text_type, PageElement):
-
- def __getattr__(self, attr):
- """text.string gives you text. This is for backwards
- compatibility for Navigable*String, but for CData* it lets you
- get the string without the CData wrapper."""
- if attr == 'string':
- return self
- else:
- raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, attr))
-
- def __unicode__(self):
- return self.__str__()
-
- def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING):
- if encoding:
- return self.encode(encoding)
- else:
- return self
-
-
-class CData(NavigableString):
-
- def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING):
- return "" % NavigableString.__str__(self, encoding)
-
-
-class ProcessingInstruction(NavigableString):
- def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING):
- output = self
- if "%SOUP-ENCODING%" in output:
- output = self.substituteEncoding(output, encoding)
- return "" % self.toEncoding(output, encoding)
-
-
-class Comment(NavigableString):
- def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING):
- return "" % NavigableString.__str__(self, encoding)
-
-
-class Declaration(NavigableString):
- def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING):
- return "" % NavigableString.__str__(self, encoding)
-
-
-class Tag(PageElement):
- """Represents a found HTML tag with its attributes and contents."""
-
- XML_ENTITIES_TO_CHARS = {'apos': "'",
- "quot": '"',
- "amp": "&",
- "lt": "<",
- "gt": ">"}
- # An RE for finding ampersands that aren't the start of of a
- # numeric entity.
- BARE_AMPERSAND = re.compile("&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)")
-
- def __init__(self, parser, name, attrs=None, parent=None,
- previous=None):
- "Basic constructor."
-
- # We don't actually store the parser object: that lets extracted
- # chunks be garbage-collected
- self.parserClass = parser.__class__
- self.isSelfClosing = parser.isSelfClosingTag(name)
- self.convertHTMLEntities = parser.convertHTMLEntities
- self.name = name
- if attrs is None:
- attrs = []
- self.attrs = attrs
- self.contents = []
- self.setup(parent, previous)
- self.hidden = False
- self.containsSubstitutions = False
-
- def get(self, key, default=None):
- """Returns the value of the 'key' attribute for the tag, or
- the value given for 'default' if it doesn't have that
- attribute."""
- return self._getAttrMap().get(key, default)
-
- def has_key(self, key):
- return key in self._getAttrMap()
-
- def __getitem__(self, key):
- """tag[key] returns the value of the 'key' attribute for the tag,
- and throws an exception if it's not there."""
- return self._getAttrMap()[key]
-
- def __iter__(self):
- "Iterating over a tag iterates over its contents."
- return iter(self.contents)
-
- def __len__(self):
- "The length of a tag is the length of its list of contents."
- return len(self.contents)
-
- def __contains__(self, x):
- return x in self.contents
-
- def __bool__(self):
- "A tag is non-None even if it has no contents."
- return True
- __nonzero__ = __bool__
-
- def __setitem__(self, key, value):
- """Setting tag[key] sets the value of the 'key' attribute for the
- tag."""
- self._getAttrMap()
- self.attrMap[key] = value
- found = False
- for i in range(0, len(self.attrs)):
- if self.attrs[i][0] == key:
- self.attrs[i] = (key, value)
- found = True
- if not found:
- self.attrs.append((key, value))
- self._getAttrMap()[key] = value
-
- def __delitem__(self, key):
- "Deleting tag[key] deletes all 'key' attributes for the tag."
- for item in self.attrs:
- if item[0] == key:
- self.attrs.remove(item)
- # We don't break because bad HTML can define the same
- # attribute multiple times.
- self._getAttrMap()
- if key in self.attrMap:
- del self.attrMap[key]
-
- def __call__(self, *args, **kwargs):
- """Calling a tag like a function is the same as calling its
- findAll() method. Eg. tag('a') returns a list of all the A tags
- found within this tag."""
- return self.findAll(*args, **kwargs)
-
- def __getattr__(self, tag):
- if len(tag) > 3 and tag.rfind('Tag') == len(tag) - 3:
- return self.find(tag[:-3])
- elif tag.find('__') != 0:
- return self.find(tag)
-
- def __eq__(self, other):
- """Returns true iff this tag has the same name, the same attributes,
- and the same contents (recursively) as the given tag.
-
- NOTE: right now this will return false if two tags have the
- same attributes in a different order. Should this be fixed?"""
- if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other):
- return False
- for i in range(0, len(self.contents)):
- if self.contents[i] != other.contents[i]:
- return False
- return True
-
- def __ne__(self, other):
- """Returns true iff this tag is not identical to the other tag,
- as defined in __eq__."""
- return not self == other
-
- def __repr__(self, encoding=DEFAULT_OUTPUT_ENCODING):
- """Renders this tag as a string."""
- return self.__str__(encoding)
-
- def __unicode__(self):
- return self.__str__(None)
-
- def _convertEntities(self, match):
- x = match.group(1)
- if x in name2codepoint:
- return six.unichr(name2codepoint[x])
- elif "&" + x + ";" in self.XML_ENTITIES_TO_CHARS:
- return '&%s;' % x
- else:
- return '&%s;' % x
-
- def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING,
- prettyPrint=False, indentLevel=0):
- """Returns a string or Unicode representation of this tag and
- its contents. To get Unicode, pass None for encoding.
-
- NOTE: since Python's HTML parser consumes whitespace, this
- method is not certain to reproduce the whitespace present in
- the original string."""
-
- encodedName = self.toEncoding(self.name, encoding)
-
- attrs = []
- if self.attrs:
- for key, val in self.attrs:
- fmt = '%s="%s"'
- if isString(val):
- if self.containsSubstitutions and '%SOUP-ENCODING%' in val:
- val = self.substituteEncoding(val, encoding)
-
- # The attribute value either:
- #
- # * Contains no embedded double quotes or single quotes.
- # No problem: we enclose it in double quotes.
- # * Contains embedded single quotes. No problem:
- # double quotes work here too.
- # * Contains embedded double quotes. No problem:
- # we enclose it in single quotes.
- # * Embeds both single _and_ double quotes. This
- # can't happen naturally, but it can happen if
- # you modify an attribute value after parsing
- # the document. Now we have a bit of a
- # problem. We solve it by enclosing the
- # attribute in single quotes, and escaping any
- # embedded single quotes to XML entities.
- if '"' in val:
- # This can't happen naturally, but it can happen
- # if you modify an attribute value after parsing.
- if "'" in val:
- val = val.replace('"', """)
- else:
- fmt = "%s='%s'"
-
- # Optionally convert any HTML entities
- if self.convertHTMLEntities:
- val = re.sub("&(\w+);", self._convertEntities, val)
-
- # Now we're okay w/r/t quotes. But the attribute
- # value might also contain angle brackets, or
- # ampersands that aren't part of entities. We need
- # to escape those to XML entities too.
- val = val.replace("<", "<").replace(">", ">")
- val = self.BARE_AMPERSAND.sub("&", val)
-
- attrs.append(fmt % (self.toEncoding(key, encoding),
- self.toEncoding(val, encoding)))
- close = ''
- closeTag = ''
- if self.isSelfClosing:
- close = ' /'
- else:
- closeTag = '' % encodedName
-
- indentTag, indentContents = 0, 0
- if prettyPrint:
- indentTag = indentLevel
- space = (' ' * (indentTag - 1))
- indentContents = indentTag + 1
- contents = self.renderContents(encoding, prettyPrint, indentContents)
- if self.hidden:
- s = contents
- else:
- s = []
- attributeString = ''
- if attrs:
- attributeString = ' ' + ' '.join(attrs)
- if prettyPrint:
- s.append(space)
- s.append('<%s%s%s>' % (encodedName, attributeString, close))
- if prettyPrint:
- s.append("\n")
- s.append(contents)
- if prettyPrint and contents and contents[-1] != "\n":
- s.append("\n")
- if prettyPrint and closeTag:
- s.append(space)
- s.append(closeTag)
- if prettyPrint and closeTag and self.nextSibling:
- s.append("\n")
- s = ''.join(s)
- return s
-
- def prettify(self, encoding=DEFAULT_OUTPUT_ENCODING):
- return self.__str__(encoding, True)
-
- def renderContents(self, encoding=DEFAULT_OUTPUT_ENCODING,
- prettyPrint=False, indentLevel=0):
- """Renders the contents of this tag as a string in the given
- encoding. If encoding is None, returns a Unicode string.."""
- s = []
- for c in self:
- text = None
- if isinstance(c, NavigableString):
- text = c.__str__(encoding)
- elif isinstance(c, Tag):
- s.append(c.__str__(encoding, prettyPrint, indentLevel))
- if text and prettyPrint:
- text = text.strip()
- if text:
- if prettyPrint:
- s.append(" " * (indentLevel - 1))
- s.append(text)
- if prettyPrint:
- s.append("\n")
- return ''.join(s)
-
- # Soup methods
-
- def find(self, name=None, attrs={}, recursive=True, text=None,
- **kwargs):
- """Return only the first child of this Tag matching the given
- criteria."""
- r = None
- l = self.findAll(name, attrs, recursive, text, 1, **kwargs)
- if l:
- r = l[0]
- return r
- findChild = find
-
- def findAll(self, name=None, attrs={}, recursive=True, text=None,
- limit=None, **kwargs):
- """Extracts a list of Tag objects that match the given
- criteria. You can specify the name of the Tag and any
- attributes you want the Tag to have.
-
- The value of a key-value pair in the 'attrs' map can be a
- string, a list of strings, a regular expression object, or a
- callable that takes a string and returns whether or not the
- string matches for some custom definition of 'matches'. The
- same is true of the tag name."""
- generator = self.recursiveChildGenerator
- if not recursive:
- generator = self.childGenerator
- return self._findAll(name, attrs, text, limit, generator, **kwargs)
- findChildren = findAll
-
- # Pre-3.x compatibility methods
- first = find
- fetch = findAll
-
- def fetchText(self, text=None, recursive=True, limit=None):
- return self.findAll(text=text, recursive=recursive, limit=limit)
-
- def firstText(self, text=None, recursive=True):
- return self.find(text=text, recursive=recursive)
-
- # Utility methods
-
- def append(self, tag):
- """Appends the given tag to the contents of this tag."""
- self.contents.append(tag)
-
- # Private methods
-
- def _getAttrMap(self):
- """Initializes a map representation of this tag's attributes,
- if not already initialized."""
- if not getattr(self, 'attrMap'):
- self.attrMap = {}
- for (key, value) in self.attrs:
- self.attrMap[key] = value
- return self.attrMap
-
- # Generator methods
- def childGenerator(self):
- for i in range(0, len(self.contents)):
- yield self.contents[i]
- raise StopIteration
-
- def recursiveChildGenerator(self):
- stack = [(self, 0)]
- while stack:
- tag, start = stack.pop()
- if isinstance(tag, Tag):
- for i in range(start, len(tag.contents)):
- a = tag.contents[i]
- yield a
- if isinstance(a, Tag) and tag.contents:
- if i < len(tag.contents) - 1:
- stack.append((tag, i + 1))
- stack.append((a, 0))
- break
- raise StopIteration
-
-
-# Next, a couple classes to represent queries and their results.
-class SoupStrainer:
- """Encapsulates a number of ways of matching a markup element (tag or
- text)."""
-
- def __init__(self, name=None, attrs={}, text=None, **kwargs):
- self.name = name
- if isString(attrs):
- kwargs['class'] = attrs
- attrs = None
- if kwargs:
- if attrs:
- attrs = attrs.copy()
- attrs.update(kwargs)
- else:
- attrs = kwargs
- self.attrs = attrs
- self.text = text
-
- def __str__(self):
- if self.text:
- return self.text
- else:
- return "%s|%s" % (self.name, self.attrs)
-
- def searchTag(self, markupName=None, markupAttrs={}):
- found = None
- markup = None
- if isinstance(markupName, Tag):
- markup = markupName
- markupAttrs = markup
- callFunctionWithTagData = callable(self.name) and \
- not isinstance(markupName, Tag)
-
- if not self.name or callFunctionWithTagData or \
- (markup and self._matches(markup, self.name)) or \
- (not markup and self._matches(markupName, self.name)):
- if callFunctionWithTagData:
- match = self.name(markupName, markupAttrs)
- else:
- match = True
- markupAttrMap = None
- for attr, matchAgainst in self.attrs.items():
- if not markupAttrMap:
- if hasattr(markupAttrs, 'get'):
- markupAttrMap = markupAttrs
- else:
- markupAttrMap = {}
- for k, v in markupAttrs:
- markupAttrMap[k] = v
- attrValue = markupAttrMap.get(attr)
- if not self._matches(attrValue, matchAgainst):
- match = False
- break
- if match:
- if markup:
- found = markup
- else:
- found = markupName
- return found
-
- def search(self, markup):
- found = None
- # If given a list of items, scan it for a text element that
- # matches.
- if isList(markup) and not isinstance(markup, Tag):
- for element in markup:
- if isinstance(element, NavigableString) and \
- self.search(element):
- found = element
- break
- # If it's a Tag, make sure its name or attributes match.
- # Don't bother with Tags if we're searching for text.
- elif isinstance(markup, Tag):
- if not self.text:
- found = self.searchTag(markup)
- # If it's text, make sure the text matches.
- elif isinstance(markup, NavigableString) or isString(markup):
- if self._matches(markup, self.text):
- found = markup
- else:
- raise Exception("I don't know how to match against a %s" %
- markup.__class__)
- return found
-
- def _matches(self, markup, matchAgainst):
- result = False
- if matchAgainst is True:
- result = markup is not None
- elif callable(matchAgainst):
- result = matchAgainst(markup)
- else:
- # Custom match methods take the tag as an argument, but all
- # other ways of matching match the tag name as a string.
- if isinstance(markup, Tag):
- markup = markup.name
- if markup and not isString(markup):
- markup = six.text_type(markup)
- # Now we know that chunk is either a string, or None.
- if hasattr(matchAgainst, 'match'):
- # It's a regexp object.
- result = markup and matchAgainst.search(markup)
- elif isList(matchAgainst):
- result = markup in matchAgainst
- elif hasattr(matchAgainst, 'items'):
- result = matchAgainst in markup
- elif matchAgainst and isString(markup):
- if isinstance(markup, six.text_type):
- matchAgainst = six.text_type(matchAgainst)
- else:
- matchAgainst = str(matchAgainst)
-
- if not result:
- result = matchAgainst == markup
- return result
-
-
-class ResultSet(list):
- """A ResultSet is just a list that keeps track of the SoupStrainer
- that created it."""
-
- def __init__(self, source):
- list.__init__([])
- self.source = source
-
-# Now, some helper functions.
-
-
-def isList(l):
- """Convenience method to determine whether or not something is listlike."""
- return hasattr(l, '__iter__') or \
- type(l) in (list, tuple)
-
-
-def isString(s):
- """Convenience method to determine whether or not something is stringlike."""
- return isinstance(s, six.string_types)
-
-
-def buildTagMap(default, *args):
- """Turns a list of maps, lists, or scalars into a single map.
- Used to build the SELF_CLOSING_TAGS, NESTABLE_TAGS, and
- NESTING_RESET_TAGS maps out of lists and partial maps."""
- built = {}
- for portion in args:
- if hasattr(portion, 'items'):
- # It's a map. Merge it.
- for k, v in portion.items():
- built[k] = v
- elif isList(portion):
- # It's a list. Map each item to the default.
- for k in portion:
- built[k] = default
- else:
- # It's a scalar. Map it to the default.
- built[portion] = default
- return built
-
-# Now, the parser classes.
-
-
-class BeautifulStoneSoup(Tag, SGMLParser):
-
- """This class contains the basic parser and search code. It defines
- a parser that knows nothing about tag behavior except for the
- following:
-
- You can't close a tag without closing all the tags it encloses.
- That is, "FooBar
should pop to 'p', not 'b'. -
Foo
| * | * should pop to 'tr', not the first 'td'
- """
-
- nestingResetTriggers = self.NESTABLE_TAGS.get(name)
- isNestable = nestingResetTriggers is not None
- isResetNesting = name in self.RESET_NESTING_TAGS
- popTo = None
- inclusive = True
- for i in range(len(self.tagStack) - 1, 0, -1):
- p = self.tagStack[i]
- if (not p or p.name == name) and not isNestable:
- # Non-nestable tags get popped to the top or to their
- # last occurance.
- popTo = name
- break
- if (nestingResetTriggers is not None and
- p.name in nestingResetTriggers) or \
- (nestingResetTriggers is None and isResetNesting and
- p.name in self.RESET_NESTING_TAGS):
- # If we encounter one of the nesting reset triggers
- # peculiar to this tag, or we encounter another tag
- # that causes nesting to reset, pop up to but not
- # including that tag.
- popTo = p.name
- inclusive = False
- break
- p = p.parent
- if popTo:
- self._popToTag(popTo, inclusive)
-
- def unknown_starttag(self, name, attrs, selfClosing=0):
- if self.quoteStack:
- # This is not a real tag.
- attrs = ''.join(' %s="%s"' % (x, y) for x, y in attrs)
- self.currentData.append('<%s%s>' % (name, attrs))
- return
- self.endData()
-
- if not self.isSelfClosingTag(name) and not selfClosing:
- self._smartPop(name)
-
- if self.parseOnlyThese and len(self.tagStack) <= 1 and \
- (self.parseOnlyThese.text or not self.parseOnlyThese.searchTag(name, attrs)):
- return
-
- tag = Tag(self, name, attrs, self.currentTag, self.previous)
- if self.previous:
- self.previous.next = tag
- self.previous = tag
- self.pushTag(tag)
- if selfClosing or self.isSelfClosingTag(name):
- self.popTag()
- if name in self.QUOTE_TAGS:
- self.quoteStack.append(name)
- self.literal = 1
- return tag
-
- def unknown_endtag(self, name):
- if self.quoteStack and self.quoteStack[-1] != name:
- # This is not a real end tag.
- self.currentData.append('' % name)
- return
- self.endData()
- self._popToTag(name)
- if self.quoteStack and self.quoteStack[-1] == name:
- self.quoteStack.pop()
- self.literal = (len(self.quoteStack) > 0)
-
- def handle_data(self, data):
- if self.convertHTMLEntities:
- if data[0] == '&':
- data = self.BARE_AMPERSAND.sub("&", data)
- else:
- data = data.replace('&', '&') \
- .replace('<', '<') \
- .replace('>', '>')
- self.currentData.append(data)
-
- def _toStringSubclass(self, text, subclass):
- """Adds a certain piece of text to the tree as a NavigableString
- subclass."""
- self.endData()
- self.handle_data(text)
- self.endData(subclass)
-
- def handle_pi(self, text):
- """Handle a processing instruction as a ProcessingInstruction
- object, possibly one with a %SOUP-ENCODING% slot into which an
- encoding will be plugged later."""
- if text[:3] == "xml":
- text = "xml version='1.0' encoding='%SOUP-ENCODING%'"
- self._toStringSubclass(text, ProcessingInstruction)
-
- def handle_comment(self, text):
- "Handle comments as Comment objects."
- self._toStringSubclass(text, Comment)
-
- def handle_charref(self, ref):
- "Handle character references as data."
- if ref[0] == 'x':
- data = six.unichr(int(ref[1:], 16))
- else:
- data = six.unichr(int(ref))
-
- if u'\x80' <= data <= u'\x9F':
- data = UnicodeDammit.subMSChar(chr(ord(data)), self.smartQuotesTo)
- elif not self.convertHTMLEntities and not self.convertXMLEntities:
- data = '&#%s;' % ref
-
- self.handle_data(data)
-
- def handle_entityref(self, ref):
- """Handle entity references as data, possibly converting known
- HTML entity references to the corresponding Unicode
- characters."""
- replaceWithXMLEntity = self.convertXMLEntities and \
- ref in self.XML_ENTITIES_TO_CHARS
- if self.convertHTMLEntities or replaceWithXMLEntity:
- try:
- data = six.unichr(name2codepoint[ref])
- except KeyError:
- if replaceWithXMLEntity:
- data = self.XML_ENTITIES_TO_CHARS.get(ref)
- else:
- data = "&%s" % ref
- else:
- data = '&%s;' % ref
- self.handle_data(data)
-
- def handle_decl(self, data):
- "Handle DOCTYPEs and the like as Declaration objects."
- self._toStringSubclass(data, Declaration)
-
- def parse_declaration(self, i):
- """Treat a bogus SGML declaration as raw data. Treat a CDATA
- declaration as a CData object."""
- j = None
- if self.rawdata[i:i + 9] == '', i)
- if k == -1:
- k = len(self.rawdata)
- data = self.rawdata[i + 9:k]
- j = k + 3
- self._toStringSubclass(data, CData)
- else:
- try:
- j = SGMLParser.parse_declaration(self, i)
- except SGMLParseError:
- toHandle = self.rawdata[i:]
- self.handle_data(toHandle)
- j = i + len(toHandle)
- return j
-
-
-class BeautifulSoup(BeautifulStoneSoup):
-
- """This parser knows the following facts about HTML:
-
- * Some tags have no closing tag and should be interpreted as being
- closed as soon as they are encountered.
-
- * The text inside some tags (ie. 'script') may contain tags which
- are not really part of the document and which should be parsed
- as text, not tags. If you want to parse the text as tags, you can
- always fetch it and parse it explicitly.
-
- * Tag nesting rules:
-
- Most tags can't be nested at all. For instance, the occurance of
- a tag should implicitly close the previous tag. - - Para1 Para2 - should be transformed into: - Para1 Para2 - - Some tags can be nested arbitrarily. For instance, the occurance - of a tag should _not_ implicitly close the previous -tag. - - Alice said:Bob said:Blah - should NOT be transformed into: - Alice said:Bob said:Blah - - Some tags can be nested, but the nesting is reset by the - interposition of other tags. For instance, a |