diff options
-rw-r--r-- | src/beautifulsoup/builder/__init__.py | 101 | ||||
-rw-r--r-- | src/beautifulsoup/builder/html5lib_builder.py | 202 | ||||
-rw-r--r-- | src/beautifulsoup/builder/lxml_builder.py | 36 | ||||
-rw-r--r-- | src/beautifulsoup/docs/__init__.py | 1 | ||||
-rw-r--r-- | src/beautifulsoup/tests/__init__.py | 1 | ||||
-rw-r--r-- | src/beautifulsoup/tests/test_docs.py | 36 | ||||
-rw-r--r-- | src/beautifulsoup/tests/test_html5lib.py | 48 | ||||
-rw-r--r-- | src/beautifulsoup/tests/test_lxml.py | 28 | ||||
-rw-r--r-- | src/beautifulsoup/tests/test_soup.py | 112 | ||||
-rw-r--r-- | src/beautifulsoup/tests/test_strainer.py | 12 | ||||
-rw-r--r-- | src/beautifulsoup/tests/test_tree.py | 743 |
11 files changed, 0 insertions, 1320 deletions
diff --git a/src/beautifulsoup/builder/__init__.py b/src/beautifulsoup/builder/__init__.py deleted file mode 100644 index fe80953..0000000 --- a/src/beautifulsoup/builder/__init__.py +++ /dev/null @@ -1,101 +0,0 @@ -from beautifulsoup.element import Entities - -__all__ = [ - 'HTMLTreeBuilder', - 'SAXTreeBuilder', - 'TreeBuilder', - ] - -class TreeBuilder(Entities): - """Turn a document into a Beautiful Soup object tree.""" - - assume_html = False - smart_quotes_to = Entities.XML_ENTITIES - convert_html_entities = False - convert_xml_entities = False - - def __init__(self): - self.soup = None - - def isSelfClosingTag(self, name): - return name in self.self_closing_tags - - def reset(self): - pass - - def feed(self, markup): - raise NotImplementedError() - - def test_fragment_to_document(self, fragment): - """Wrap an HTML fragment to make it look like a document. - - Different parsers do this differently. For instance, lxml - introduces an empty <head> tag, and html5lib - doesn't. Abstracting this away lets us write simple tests - which run HTML fragments through the parser and compare the - results against other HTML fragments. - - This method should not be used outside of tests. - """ - return fragment - - -class SAXTreeBuilder(TreeBuilder): - """A Beautiful Soup treebuilder that listens for SAX events.""" - - def feed(self, markup): - raise NotImplementedError() - - def close(self): - pass - - def startElement(self, name, attrs): - attrs = dict((key[1], value) for key, value in attrs.items()) - #print "Start %s, %r" % (name, attrs) - self.soup.handle_starttag(name, attrs) - - def endElement(self, name): - #print "End %s" % name - self.soup.handle_endtag(name) - - def startElementNS(self, nsTuple, nodeName, attrs): - # Throw away (ns, nodeName) for now. - self.startElement(nodeName, attrs) - - def endElementNS(self, nsTuple, nodeName): - # Throw away (ns, nodeName) for now. - self.endElement(nodeName) - #handler.endElementNS((ns, node.nodeName), node.nodeName) - - def startPrefixMapping(self, prefix, nodeValue): - # Ignore the prefix for now. - pass - - def endPrefixMapping(self, prefix): - # Ignore the prefix for now. - # handler.endPrefixMapping(prefix) - pass - - def characters(self, content): - self.soup.handle_data(content) - - def startDocument(self): - pass - - def endDocument(self): - pass - - -class HTMLTreeBuilder(TreeBuilder): - """This TreeBuilder knows facts about HTML. - - Such as which tags are self-closing tags. - """ - - assume_html = True - smart_quotes_to = Entities.HTML_ENTITIES - - preserve_whitespace_tags = set(['pre', 'textarea']) - self_closing_tags = set(['br' , 'hr', 'input', 'img', 'meta', - 'spacer', 'link', 'frame', 'base']) - diff --git a/src/beautifulsoup/builder/html5lib_builder.py b/src/beautifulsoup/builder/html5lib_builder.py deleted file mode 100644 index 0a4b9e7..0000000 --- a/src/beautifulsoup/builder/html5lib_builder.py +++ /dev/null @@ -1,202 +0,0 @@ -from beautifulsoup.builder import HTMLTreeBuilder, SAXTreeBuilder -import html5lib -from html5lib.constants import DataLossWarning -import warnings -from beautifulsoup.element import Tag, NavigableString, Comment, Declaration - - -class HTML5TreeBuilder(HTMLTreeBuilder): - """Use html5lib to build a tree.""" - - # These methods are defined by Beautiful Soup. - def feed(self, markup): - parser = html5lib.HTMLParser(tree=self.create_treebuilder) - doc = parser.parse(markup) - - def create_treebuilder(self, namespaceHTMLElements): - self.underlying_builder = TreeBuilderForHtml5lib( - self.soup, namespaceHTMLElements) - return self.underlying_builder - - def test_fragment_to_document(self, fragment): - """See `TreeBuilder`.""" - return u'<html><head></head><body>%s</body></html>' % fragment - - -class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder): - - def __init__(self, soup, namespaceHTMLElements): - self.soup = soup - if namespaceHTMLElements: - warnings.warn("namespaceHTMLElements not supported yet", DataLossWarning) - super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements) - - def documentClass(self): - self.soup.reset() - return Element(self.soup, self.soup, None) - - def insertDoctype(self, token): - name = token["name"] - publicId = token["publicId"] - systemId = token["systemId"] - - if publicId: - self.soup.insert(0, Declaration("%s PUBLIC \"%s\" \"%s\""%(name, publicId, systemId or ""))) - elif systemId: - self.soup.insert(0, Declaration("%s SYSTEM \"%s\""% - (name, systemId))) - else: - self.soup.insert(0, Declaration(name)) - - def elementClass(self, name, namespace): - if namespace is not None: - warnings.warn("BeautifulSoup cannot represent elements in any namespace", DataLossWarning) - return Element(Tag(self.soup, self.soup.builder, name), self.soup, namespace) - - def commentClass(self, data): - return TextNode(Comment(data), self.soup) - - def fragmentClass(self): - self.soup = BeautifulSoup("") - self.soup.name = "[document_fragment]" - return Element(self.soup, self.soup, None) - - def appendChild(self, node): - self.soup.insert(len(self.soup.contents), node.element) - - def testSerializer(self, element): - return testSerializer(element) - - def getDocument(self): - return self.soup - - def getFragment(self): - return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element - -class AttrList(object): - def __init__(self, element): - self.element = element - self.attrs = dict(self.element.attrs) - def __iter__(self): - return self.attrs.items().__iter__() - def __setitem__(self, name, value): - "set attr", name, value - self.element[name] = value - def items(self): - return self.attrs.items() - def keys(self): - return self.attrs.keys() - def __getitem__(self, name): - return self.attrs[name] - def __contains__(self, name): - return name in self.attrs.keys() - - -class Element(html5lib.treebuilders._base.Node): - def __init__(self, element, soup, namespace): - html5lib.treebuilders._base.Node.__init__(self, element.name) - self.element = element - self.soup = soup - self.namespace = namespace - - def _nodeIndex(self, node, refNode): - # Finds a node by identity rather than equality - for index in range(len(self.element.contents)): - if id(self.element.contents[index]) == id(refNode.element): - return index - return None - - def appendChild(self, node): - if (node.element.__class__ == NavigableString and self.element.contents - and self.element.contents[-1].__class__ == NavigableString): - # Concatenate new text onto old text node - # (TODO: This has O(n^2) performance, for input like "a</a>a</a>a</a>...") - newStr = NavigableString(self.element.contents[-1]+node.element) - - # Remove the old text node - # (Can't simply use .extract() by itself, because it fails if - # an equal text node exists within the parent node) - oldElement = self.element.contents[-1] - del self.element.contents[-1] - oldElement.parent = None - oldElement.extract() - - self.element.insert(len(self.element.contents), newStr) - else: - self.element.insert(len(self.element.contents), node.element) - node.parent = self - - def getAttributes(self): - return AttrList(self.element) - - def setAttributes(self, attributes): - if attributes: - for name, value in attributes.items(): - self.element[name] = value - - attributes = property(getAttributes, setAttributes) - - def insertText(self, data, insertBefore=None): - text = TextNode(NavigableString(data), self.soup) - if insertBefore: - self.insertBefore(text, insertBefore) - else: - self.appendChild(text) - - def insertBefore(self, node, refNode): - index = self._nodeIndex(node, refNode) - if (node.element.__class__ == NavigableString and self.element.contents - and self.element.contents[index-1].__class__ == NavigableString): - # (See comments in appendChild) - newStr = NavigableString(self.element.contents[index-1]+node.element) - oldNode = self.element.contents[index-1] - del self.element.contents[index-1] - oldNode.parent = None - oldNode.extract() - - self.element.insert(index-1, newStr) - else: - self.element.insert(index, node.element) - node.parent = self - - def removeChild(self, node): - index = self._nodeIndex(node.parent, node) - del node.parent.element.contents[index] - node.element.parent = None - node.element.extract() - node.parent = None - - def reparentChildren(self, newParent): - while self.element.contents: - child = self.element.contents[0] - child.extract() - if isinstance(child, Tag): - newParent.appendChild(Element(child, self.soup, namespaces["html"])) - else: - newParent.appendChild(TextNode(child, self.soup)) - - def cloneNode(self): - node = Element(Tag(self.soup, self.soup.builder, self.element.name), self.soup, self.namespace) - for key,value in self.attributes: - node.attributes[key] = value - return node - - def hasContent(self): - return self.element.contents - - def getNameTuple(self): - if self.namespace == None: - return namespaces["html"], self.name - else: - return self.namespace, self.name - - nameTuple = property(getNameTuple) - -class TextNode(Element): - def __init__(self, element, soup): - html5lib.treebuilders._base.Node.__init__(self, None) - self.element = element - self.soup = soup - - def cloneNode(self): - raise NotImplementedError diff --git a/src/beautifulsoup/builder/lxml_builder.py b/src/beautifulsoup/builder/lxml_builder.py deleted file mode 100644 index 3e1de5f..0000000 --- a/src/beautifulsoup/builder/lxml_builder.py +++ /dev/null @@ -1,36 +0,0 @@ -from lxml import etree -from beautifulsoup.element import Comment -from beautifulsoup.builder import HTMLTreeBuilder - -class LXMLTreeBuilder(HTMLTreeBuilder): - - def __init__(self, parser_class=etree.HTMLParser): - self.parser = parser_class(target=self) - self.soup = None - - def feed(self, markup): - self.parser.feed(markup) - self.parser.close() - - def close(self): - pass - - def start(self, name, attrs): - self.soup.handle_starttag(name, attrs) - - def end(self, name): - self.soup.handle_endtag(name) - - def data(self, content): - self.soup.handle_data(content) - - def comment(self, content): - "Handle comments as Comment objects." - self.soup.endData() - self.soup.handle_data(content) - self.soup.endData(Comment) - - def test_fragment_to_document(self, fragment): - """See `TreeBuilder`.""" - return u'<html><body>%s</body></html>' % fragment - diff --git a/src/beautifulsoup/docs/__init__.py b/src/beautifulsoup/docs/__init__.py deleted file mode 100644 index a2920fe..0000000 --- a/src/beautifulsoup/docs/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Executable documentation about beautifulsoup.""" diff --git a/src/beautifulsoup/tests/__init__.py b/src/beautifulsoup/tests/__init__.py deleted file mode 100644 index 142c8cc..0000000 --- a/src/beautifulsoup/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"The beautifulsoup tests." diff --git a/src/beautifulsoup/tests/test_docs.py b/src/beautifulsoup/tests/test_docs.py deleted file mode 100644 index 58316d6..0000000 --- a/src/beautifulsoup/tests/test_docs.py +++ /dev/null @@ -1,36 +0,0 @@ -"Test harness for doctests." - -# pylint: disable-msg=E0611,W0142 - -__metaclass__ = type -__all__ = [ - 'additional_tests', - ] - -import atexit -import doctest -import os -from pkg_resources import ( - resource_filename, resource_exists, resource_listdir, cleanup_resources) -import unittest - -DOCTEST_FLAGS = ( - doctest.ELLIPSIS | - doctest.NORMALIZE_WHITESPACE | - doctest.REPORT_NDIFF) - - -def additional_tests(): - "Run the doc tests (README.txt and docs/*, if any exist)" - doctest_files = [ - os.path.abspath(resource_filename('beautifulsoup', 'README.txt'))] - if resource_exists('beautifulsoup', 'docs'): - for name in resource_listdir('beautifulsoup', 'docs'): - if name.endswith('.txt'): - doctest_files.append( - os.path.abspath( - resource_filename('beautifulsoup', 'docs/%s' % name))) - kwargs = dict(module_relative=False, optionflags=DOCTEST_FLAGS) - atexit.register(cleanup_resources) - return unittest.TestSuite(( - doctest.DocFileSuite(*doctest_files, **kwargs))) diff --git a/src/beautifulsoup/tests/test_html5lib.py b/src/beautifulsoup/tests/test_html5lib.py deleted file mode 100644 index 131c999..0000000 --- a/src/beautifulsoup/tests/test_html5lib.py +++ /dev/null @@ -1,48 +0,0 @@ -from beautifulsoup.builder.html5lib_builder import HTML5TreeBuilder -from beautifulsoup.testing import ( - BuilderInvalidMarkupSmokeTest, - BuilderSmokeTest, -) - - -class TestHTML5Builder(BuilderSmokeTest): - """See `BuilderSmokeTest`.""" - - def setUp(self): - self.default_builder = HTML5TreeBuilder() - - def test_collapsed_whitespace(self): - """Whitespace is preserved even in tags that don't require it.""" - self.assertSoupEquals("<p> </p>") - self.assertSoupEquals("<b> </b>") - - -class TestHTML5BuilderInvalidMarkup(BuilderInvalidMarkupSmokeTest): - """See `BuilderInvalidMarkupSmokeTest`.""" - - def setUp(self): - self.default_builder = HTML5TreeBuilder() - - def test_unclosed_block_level_elements(self): - # The unclosed <b> tag is closed so that the block-level tag - # can be closed, and another <b> tag is inserted after the - # next block-level tag begins. - self.assertSoupEquals( - '<blockquote><p><b>Foo</blockquote><p>Bar', - '<blockquote><p><b>Foo</b></p></blockquote><p><b>Bar</b></p>') - - def test_incorrectly_nested_tables(self): - self.assertSoupEquals( - '<table><tr><table><tr id="nested">', - ('<table><tbody><tr></tr></tbody></table>' - '<table><tbody><tr id="nested"></tr></tbody></table>')) - - - def test_foo(self): - isolatin = """<html><meta http-equiv="Content-type" content="text/html; charset=ISO-Latin-1" />Sacr\xe9 bleu!</html>""" - soup = self.soup(isolatin) - - utf8 = isolatin.replace("ISO-Latin-1".encode(), "utf-8".encode()) - utf8 = utf8.replace("\xe9", "\xc3\xa9") - - print soup diff --git a/src/beautifulsoup/tests/test_lxml.py b/src/beautifulsoup/tests/test_lxml.py deleted file mode 100644 index b53ee42..0000000 --- a/src/beautifulsoup/tests/test_lxml.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Tests to ensure that the lxml tree builder generates good trees.""" - -from beautifulsoup.testing import ( - BuilderInvalidMarkupSmokeTest, - BuilderSmokeTest, -) - -class TestLXMLBuilder(BuilderSmokeTest): - """See `BuilderSmokeTest`.""" - - def test_bare_string(self): - # lxml puts a <p> tag around the bare string. - self.assertSoupEquals( - "A bare string", "<p>A bare string</p>") - - def test_foo(self): - isolatin = """<html><meta http-equiv="Content-type" content="text/html; charset=ISO-Latin-1" />Sacr\xe9 bleu!</html>""" - soup = self.soup(isolatin) - - utf8 = isolatin.replace("ISO-Latin-1".encode(), "utf-8".encode()) - utf8 = utf8.replace("\xe9", "\xc3\xa9") - - print soup - - -class TestLXMLBuilderInvalidMarkup(BuilderInvalidMarkupSmokeTest): - """See `BuilderInvalidMarkupSmokeTest`.""" - diff --git a/src/beautifulsoup/tests/test_soup.py b/src/beautifulsoup/tests/test_soup.py deleted file mode 100644 index ec0394d..0000000 --- a/src/beautifulsoup/tests/test_soup.py +++ /dev/null @@ -1,112 +0,0 @@ -# -*- coding: utf-8 -*- -"""Tests of Beautiful Soup as a whole.""" - -import unittest -from beautifulsoup.element import SoupStrainer -from beautifulsoup.dammit import UnicodeDammit -from beautifulsoup.testing import SoupTest - - -class TestEncodingConversion(SoupTest): - # Test Beautiful Soup's ability to decode and encode from various - # encodings. - - def setUp(self): - super(TestEncodingConversion, self).setUp() - self.unicode_data = u"<html><body><foo>\xe9</foo></body></html>" - self.utf8_data = self.unicode_data.encode("utf-8") - self.assertEqual( - self.utf8_data, "<html><body><foo>\xc3\xa9</foo></body></html>") - - def test_ascii_in_unicode_out(self): - # ASCII input is converted to Unicode. The originalEncoding - # attribute is set. - ascii = "<foo>a</foo>" - soup_from_ascii = self.soup(ascii) - unicode_output = soup_from_ascii.decode() - self.assertTrue(isinstance(unicode_output, unicode)) - self.assertEquals(unicode_output, self.document_for(ascii)) - self.assertEquals(soup_from_ascii.originalEncoding, "ascii") - - def test_unicode_in_unicode_out(self): - # Unicode input is left alone. The originalEncoding attribute - # is not set. - soup_from_unicode = self.soup(self.unicode_data) - self.assertEquals(soup_from_unicode.decode(), self.unicode_data) - self.assertEquals(soup_from_unicode.foo.string, u'\xe9') - self.assertEquals(soup_from_unicode.originalEncoding, None) - - def test_utf8_in_unicode_out(self): - # UTF-8 input is converted to Unicode. The originalEncoding - # attribute is set. - soup_from_utf8 = self.soup(self.utf8_data) - self.assertEquals(soup_from_utf8.decode(), self.unicode_data) - self.assertEquals(soup_from_utf8.foo.string, u'\xe9') - - def test_utf8_out(self): - # The internal data structures can be encoded as UTF-8. - soup_from_unicode = self.soup(self.unicode_data) - self.assertEquals(soup_from_unicode.encode('utf-8'), self.utf8_data) - - def test_hebrew(self): - # A real-world test to make sure we can convert ISO-8859-9 (a - # Hebrew encoding) to UTF-8. - iso_8859_8= '<HTML><HEAD><TITLE>Hebrew (ISO 8859-8) in Visual Directionality</TITLE></HEAD><BODY><H1>Hebrew (ISO 8859-8) in Visual Directionality</H1>\xed\xe5\xec\xf9</BODY></HTML>' - utf8 = '<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xd7\x9d\xd7\x95\xd7\x9c\xd7\xa9</body></html>' - soup = self.soup(iso_8859_8, fromEncoding="iso-8859-8") - self.assertEquals(soup.originalEncoding, 'iso-8859-8') - self.assertEquals(soup.encode('utf-8'), utf8) - - -class TestSelectiveParsing(SoupTest): - - def test_parse_with_soupstrainer(self): - markup = "No<b>Yes</b><a>No<b>Yes <c>Yes</c></b>" - strainer = SoupStrainer("b") - soup = self.soup(markup, parseOnlyThese=strainer) - self.assertEquals(soup.encode(), "<b>Yes</b><b>Yes <c>Yes</c></b>") - - -class TestUnicodeDammit(unittest.TestCase): - """Standalone tests of Unicode, Dammit.""" - - def test_smart_quotes_to_xml_entities(self): - markup = "<foo>\x91\x92\x93\x94</foo>" - dammit = UnicodeDammit(markup) - self.assertEquals( - dammit.unicode, "<foo>‘’“”</foo>") - - def test_smart_quotes_to_html_entities(self): - markup = "<foo>\x91\x92\x93\x94</foo>" - dammit = UnicodeDammit(markup, smartQuotesTo="html") - self.assertEquals( - dammit.unicode, "<foo>‘’“”</foo>") - - def test_detect_utf8(self): - utf8 = "\xc3\xa9" - dammit = UnicodeDammit(utf8) - self.assertEquals(dammit.unicode, u'\xe9') - self.assertEquals(dammit.originalEncoding, 'utf-8') - - def test_convert_hebrew(self): - hebrew = "\xed\xe5\xec\xf9" - dammit = UnicodeDammit(hebrew, ["iso-8859-8"]) - self.assertEquals(dammit.originalEncoding, 'iso-8859-8') - self.assertEquals(dammit.unicode, u'\u05dd\u05d5\u05dc\u05e9') - - def test_dont_see_smart_quotes_where_there_are_none(self): - utf_8 = "\343\202\261\343\203\274\343\202\277\343\202\244 Watch" - dammit = UnicodeDammit(utf_8) - self.assertEquals(dammit.originalEncoding, 'utf-8') - self.assertEquals(dammit.unicode.encode("utf-8"), utf_8) - - def test_ignore_inappropriate_codecs(self): - utf8_data = u"Räksmörgås".encode("utf-8") - dammit = UnicodeDammit(utf8_data, ["iso-8859-8"]) - self.assertEquals(dammit.originalEncoding, 'utf-8') - - def test_ignore_invalid_codecs(self): - utf8_data = u"Räksmörgås".encode("utf-8") - for bad_encoding in ['.utf8', '...', 'utF---16.!']: - dammit = UnicodeDammit(utf8_data, [bad_encoding]) - self.assertEquals(dammit.originalEncoding, 'utf-8') diff --git a/src/beautifulsoup/tests/test_strainer.py b/src/beautifulsoup/tests/test_strainer.py deleted file mode 100644 index f078935..0000000 --- a/src/beautifulsoup/tests/test_strainer.py +++ /dev/null @@ -1,12 +0,0 @@ -import unittest -from beautifulsoup import BeautifulSoup -from beautifulsoup.element import SoupStrainer -from beautifulsoup.testing import SoupTest - -class TestSoupStrainer(unittest.TestCase): - - def test_soupstrainer(self): - strainer = SoupStrainer("b") - soup = BeautifulSoup("A <b>bold</b> <meta /> <i>statement</i>", - parseOnlyThese=strainer) - self.assertEquals(soup.decode(), "<b>bold</b>") diff --git a/src/beautifulsoup/tests/test_tree.py b/src/beautifulsoup/tests/test_tree.py deleted file mode 100644 index 344a462..0000000 --- a/src/beautifulsoup/tests/test_tree.py +++ /dev/null @@ -1,743 +0,0 @@ -# -*- coding: utf-8 -*- -"""Tests for Beautiful Soup's tree traversal methods. - -The tree traversal methods are the main advantage of using Beautiful -Soup over other parsers. - -Different parsers will build different Beautiful Soup trees given the -same markup, but all Beautiful Soup trees can be traversed with the -methods tested here. -""" - -import re -from beautifulsoup import BeautifulSoup -from beautifulsoup.element import SoupStrainer, Tag -from beautifulsoup.testing import SoupTest - -class TreeTest(SoupTest): - - def assertSelects(self, tags, should_match): - """Make sure that the given tags have the correct text. - - This is used in tests that define a bunch of tags, each - containing a single string, and then select certain strings by - some mechanism. - """ - self.assertEqual([tag.string for tag in tags], should_match) - - def assertSelectsIDs(self, tags, should_match): - """Make sure that the given tags have the correct IDs. - - This is used in tests that define a bunch of tags, each - containing a single string, and then select certain strings by - some mechanism. - """ - self.assertEqual([tag['id'] for tag in tags], should_match) - - -class TestFind(TreeTest): - """Basic tests of the find() method. - - find() just calls findAll() with limit=1, so it's not tested all - that thouroughly here. - """ - - def test_find_tag(self): - soup = self.soup("<a>1</a><b>2</b><a>3</a><b>4</b>") - self.assertEqual(soup.find("b").string, "2") - - def test_unicode_text_find(self): - soup = self.soup(u'<h1>Räksmörgås</h1>') - self.assertEqual(soup.find(text=u'Räksmörgås'), u'Räksmörgås') - - -class TestFindAll(TreeTest): - """Basic tests of the findAll() method.""" - - def test_find_all_text_nodes(self): - """You can search the tree for text nodes.""" - soup = self.soup("<html>Foo<b>bar</b>\xbb</html>") - # Exact match. - self.assertEqual(soup.findAll(text="bar"), [u"bar"]) - # Match any of a number of strings. - self.assertEqual( - soup.findAll(text=["Foo", "bar"]), [u"Foo", u"bar"]) - # Match a regular expression. - self.assertEqual(soup.findAll(text=re.compile('.*')), - [u"Foo", u"bar", u'\xbb']) - # Match anything. - self.assertEqual(soup.findAll(text=True), - [u"Foo", u"bar", u'\xbb']) - - def test_find_all_limit(self): - """You can limit the number of items returned by findAll.""" - soup = self.soup("<a>1</a><a>2</a><a>3</a><a>4</a><a>5</a>") - self.assertSelects(soup.findAll('a', limit=3), ["1", "2", "3"]) - self.assertSelects(soup.findAll('a', limit=1), ["1"]) - self.assertSelects( - soup.findAll('a', limit=10), ["1", "2", "3", "4", "5"]) - - # A limit of 0 means no limit. - self.assertSelects( - soup.findAll('a', limit=0), ["1", "2", "3", "4", "5"]) - -class TestFindAllByName(TreeTest): - """Test ways of finding tags by tag name.""" - - def setUp(self): - super(TreeTest, self).setUp() - self.tree = self.soup("""<a>First tag.</a> - <b>Second tag.</b> - <c>Third <a>Nested tag.</a> tag.</c>""") - - def test_find_all_by_tag_name(self): - # Find all the <a> tags. - self.assertSelects( - self.tree.findAll('a'), ['First tag.', 'Nested tag.']) - - def test_find_all_on_non_root_element(self): - # You can call find_all on any node, not just the root. - self.assertSelects(self.tree.c.findAll('a'), ['Nested tag.']) - - def test_calling_element_invokes_find_all(self): - self.assertSelects(self.tree('a'), ['First tag.', 'Nested tag.']) - - def test_find_all_by_tag_strainer(self): - self.assertSelects( - self.tree.findAll(SoupStrainer('a')), - ['First tag.', 'Nested tag.']) - - def test_find_all_by_tag_names(self): - self.assertSelects( - self.tree.findAll(['a', 'b']), - ['First tag.', 'Second tag.', 'Nested tag.']) - - def test_find_all_by_tag_dict(self): - self.assertSelects( - self.tree.findAll({'a' : True, 'b' : True}), - ['First tag.', 'Second tag.', 'Nested tag.']) - - def test_find_all_by_tag_re(self): - self.assertSelects( - self.tree.findAll(re.compile('^[ab]$')), - ['First tag.', 'Second tag.', 'Nested tag.']) - - def test_find_all_with_tags_matching_method(self): - # You can define an oracle method that determines whether - # a tag matches the search. - def id_matches_name(tag): - return tag.name == tag.get('id') - - tree = self.soup("""<a id="a">Match 1.</a> - <a id="1">Does not match.</a> - <b id="b">Match 2.</a>""") - - self.assertSelects( - tree.findAll(id_matches_name), ["Match 1.", "Match 2."]) - - -class TestFindAllByAttribute(TreeTest): - - def test_find_all_by_attribute_name(self): - # You can pass in keyword arguments to findAll to search by - # attribute. - tree = self.soup(""" - <a id="first">Matching a.</a> - <a id="second"> - Non-matching <b id="first">Matching b.</b>a. - </a>""") - self.assertSelects(tree.findAll(id='first'), - ["Matching a.", "Matching b."]) - - def test_find_all_by_attribute_dict(self): - # You can pass in a dictionary as the argument 'attrs'. This - # lets you search for attributes like 'name' (a fixed argument - # to findAll) and 'class' (a reserved word in Python.) - tree = self.soup(""" - <a name="name1" class="class1">Name match.</a> - <a name="name2" class="class2">Class match.</a> - <a name="name3" class="class3">Non-match.</a> - <name1>A tag called 'name1'.</name1> - """) - - # This doesn't do what you want. - self.assertSelects(tree.findAll(name='name1'), - ["A tag called 'name1'."]) - # This does what you want. - self.assertSelects(tree.findAll(attrs={'name' : 'name1'}), - ["Name match."]) - - # Passing class='class2' would cause a syntax error. - self.assertSelects(tree.findAll(attrs={'class' : 'class2'}), - ["Class match."]) - - def test_find_all_by_class(self): - # Passing in a string to 'attrs' will search the CSS class. - tree = self.soup(""" - <a class="1">Class 1.</a> - <a class="2">Class 2.</a> - <b class="1">Class 1.</a> - """) - self.assertSelects(tree.findAll('a', '1'), ['Class 1.']) - self.assertSelects(tree.findAll(attrs='1'), ['Class 1.', 'Class 1.']) - - def test_find_all_by_attribute_soupstrainer(self): - tree = self.soup(""" - <a id="first">Match.</a> - <a id="second">Non-match.</a>""") - - strainer = SoupStrainer(attrs={'id' : 'first'}) - self.assertSelects(tree.findAll(strainer), ['Match.']) - - def test_find_all_with_missing_atribute(self): - # You can pass in None as the value of an attribute to findAll. - # This will match tags that do not have that attribute set. - tree = self.soup("""<a id="1">ID present.</a> - <a>No ID present.</a> - <a id="">ID is empty.</a>""") - self.assertSelects(tree.findAll('a', id=None), ["No ID present."]) - - def test_find_all_with_defined_attribute(self): - # You can pass in None as the value of an attribute to findAll. - # This will match tags that have that attribute set to any value. - tree = self.soup("""<a id="1">ID present.</a> - <a>No ID present.</a> - <a id="">ID is empty.</a>""") - self.assertSelects( - tree.findAll(id=True), ["ID present.", "ID is empty."]) - - def test_find_all_with_numeric_attribute(self): - # If you search for a number, it's treated as a string. - tree = self.soup("""<a id=1>Unquoted attribute.</a> - <a id="1">Quoted attribute.</a>""") - - expected = ["Unquoted attribute.", "Quoted attribute."] - self.assertSelects(tree.findAll(id=1), expected) - self.assertSelects(tree.findAll(id="1"), expected) - - def test_find_all_with_list_attribute_values(self): - # You can pass a list of attribute values instead of just one, - # and you'll get tags that match any of the values. - tree = self.soup("""<a id="1">1</a> - <a id="2">2</a> - <a id="3">3</a> - <a>No ID.</a>""") - self.assertSelects(tree.findAll(id=["1", "3", "4"]), - ["1", "3"]) - - def test_find_all_with_regular_expression_attribute_value(self): - # You can pass a regular expression as an attribute value, and - # you'll get tags whose values for that attribute match the - # regular expression. - tree = self.soup("""<a id="a">One a.</a> - <a id="aa">Two as.</a> - <a id="ab">Mixed as and bs.</a> - <a id="b">One b.</a> - <a>No ID.</a>""") - - self.assertSelects(tree.findAll(id=re.compile("^a+$")), - ["One a.", "Two as."]) - - -class TestParentOperations(TreeTest): - """Test navigation and searching through an element's parents.""" - - def setUp(self): - super(TestParentOperations, self).setUp() - self.tree = self.soup('''<ul id="empty"></ul> - <ul id="top"> - <ul id="middle"> - <ul id="bottom"> - <b>Start here</b> - </ul> - </ul>''') - self.start = self.tree.b - - - def test_parent(self): - self.assertEquals(self.start.parent['id'], 'bottom') - self.assertEquals(self.start.parent.parent['id'], 'middle') - self.assertEquals(self.start.parent.parent.parent['id'], 'top') - - def test_parent_of_top_tag_is_soup_object(self): - top_tag = self.tree.contents[0] - self.assertEquals(top_tag.parent, self.tree) - - def test_soup_object_has_no_parent(self): - self.assertEquals(None, self.tree.parent) - - def test_find_parents(self): - self.assertSelectsIDs( - self.start.findParents('ul'), ['bottom', 'middle', 'top']) - self.assertSelectsIDs( - self.start.findParents('ul', id="middle"), ['middle']) - - def test_find_parent(self): - self.assertEquals(self.start.findParent('ul')['id'], 'bottom') - - def test_parent_of_text_element(self): - text = self.tree.find(text="Start here") - self.assertEquals(text.parent.name, 'b') - - def test_text_element_find_parent(self): - text = self.tree.find(text="Start here") - self.assertEquals(text.findParent('ul')['id'], 'bottom') - - def test_parent_generator(self): - parents = [parent['id'] for parent in self.start.parentGenerator() - if parent is not None and parent.has_key('id')] - self.assertEquals(parents, ['bottom', 'middle', 'top']) - - -class ProximityTest(TreeTest): - - def setUp(self): - super(TreeTest, self).setUp() - self.tree = self.soup( - '<html id="start"><head></head><body><b id="1">One</b><b id="2">Two</b><b id="3">Three</b></body></html>') - - -class TestNextOperations(ProximityTest): - - def setUp(self): - super(TestNextOperations, self).setUp() - self.start = self.tree.b - - def test_next(self): - self.assertEquals(self.start.next, "One") - self.assertEquals(self.start.next.next['id'], "2") - - def test_next_of_last_item_is_none(self): - last = self.tree.find(text="Three") - self.assertEquals(last.next, None) - - def test_next_of_root_is_none(self): - # The document root is outside the next/previous chain. - self.assertEquals(self.tree.next, None) - - def test_find_all_next(self): - self.assertSelects(self.start.findAllNext('b'), ["Two", "Three"]) - self.assertSelects(self.start.findAllNext(id=3), ["Three"]) - - def test_find_next(self): - self.assertEquals(self.start.findNext('b')['id'], '2') - self.assertEquals(self.start.findNext(text="Three"), "Three") - - def test_find_next_for_text_element(self): - text = self.tree.find(text="One") - self.assertEquals(text.findNext("b").string, "Two") - self.assertSelects(text.findAllNext("b"), ["Two", "Three"]) - - def test_next_generator(self): - start = self.tree.find(text="Two") - successors = [node for node in start.nextGenerator()] - # There are two successors: the final <b> tag and its text contents. - # Then we go off the end. - tag, contents, none = successors - self.assertEquals(tag['id'], '3') - self.assertEquals(contents, "Three") - self.assertEquals(none, None) - - # XXX Should nextGenerator really return None? Seems like it - # should just stop. - - -class TestPreviousOperations(ProximityTest): - - def setUp(self): - super(TestPreviousOperations, self).setUp() - self.end = self.tree.find(text="Three") - - def test_previous(self): - self.assertEquals(self.end.previous['id'], "3") - self.assertEquals(self.end.previous.previous, "Two") - - def test_previous_of_first_item_is_none(self): - first = self.tree.find('html') - self.assertEquals(first.previous, None) - - def test_previous_of_root_is_none(self): - # The document root is outside the next/previous chain. - # XXX This is broken! - #self.assertEquals(self.tree.previous, None) - pass - - def test_find_all_previous(self): - # The <b> tag containing the "Three" node is the predecessor - # of the "Three" node itself, which is why "Three" shows up - # here. - self.assertSelects( - self.end.findAllPrevious('b'), ["Three", "Two", "One"]) - self.assertSelects(self.end.findAllPrevious(id=1), ["One"]) - - def test_find_previous(self): - self.assertEquals(self.end.findPrevious('b')['id'], '3') - self.assertEquals(self.end.findPrevious(text="One"), "One") - - def test_find_previous_for_text_element(self): - text = self.tree.find(text="Three") - self.assertEquals(text.findPrevious("b").string, "Three") - self.assertSelects( - text.findAllPrevious("b"), ["Three", "Two", "One"]) - - def test_previous_generator(self): - start = self.tree.find(text="One") - predecessors = [node for node in start.previousGenerator()] - - # There are four predecessors: the <b> tag containing "One" - # the <body> tag, the <head> tag, and the <html> tag. Then we - # go off the end. - b, body, head, html, none = predecessors - self.assertEquals(b['id'], '1') - self.assertEquals(body.name, "body") - self.assertEquals(head.name, "head") - self.assertEquals(html.name, "html") - self.assertEquals(none, None) - - # Again, we shouldn't be returning None. - - -class SiblingTest(TreeTest): - - def setUp(self): - super(SiblingTest, self).setUp() - markup = '''<html> - <span id="1"> - <span id="1.1"></span> - </span> - <span id="2"> - <span id="2.1"></span> - </span> - <span id="3"> - <span id="3.1"></span> - </span> - <span id="4"></span> - </html>''' - # All that whitespace looks good but makes the tests more - # difficult. Get rid of it. - markup = re.compile("\n\s*").sub("", markup) - self.tree = self.soup(markup) - - -class TestNextSibling(SiblingTest): - - def setUp(self): - super(TestNextSibling, self).setUp() - self.start = self.tree.find(id="1") - - def test_next_sibling_of_root_is_none(self): - self.assertEquals(self.tree.nextSibling, None) - - def test_next_sibling(self): - self.assertEquals(self.start.nextSibling['id'], '2') - self.assertEquals(self.start.nextSibling.nextSibling['id'], '3') - - # Note the difference between nextSibling and next. - self.assertEquals(self.start.next['id'], '1.1') - - def test_next_sibling_may_not_exist(self): - self.assertEquals(self.tree.html.nextSibling, None) - - nested_span = self.tree.find(id="1.1") - self.assertEquals(nested_span.nextSibling, None) - - last_span = self.tree.find(id="4") - self.assertEquals(last_span.nextSibling, None) - - def test_find_next_sibling(self): - self.assertEquals(self.start.findNextSibling('span')['id'], '2') - - def test_next_siblings(self): - self.assertSelectsIDs(self.start.findNextSiblings("span"), - ['2', '3', '4']) - - self.assertSelectsIDs(self.start.findNextSiblings(id='3'), ['3']) - - def test_next_sibling_for_text_element(self): - soup = self.soup("Foo<b>bar</b>baz") - start = soup.find(text="Foo") - self.assertEquals(start.nextSibling.name, 'b') - self.assertEquals(start.nextSibling.nextSibling, 'baz') - - self.assertSelects(start.findNextSiblings('b'), ['bar']) - self.assertEquals(start.findNextSibling(text="baz"), "baz") - self.assertEquals(start.findNextSibling(text="nonesuch"), None) - - -class TestPreviousSibling(SiblingTest): - - def setUp(self): - super(TestPreviousSibling, self).setUp() - self.end = self.tree.find(id="4") - - def test_previous_sibling_of_root_is_none(self): - self.assertEquals(self.tree.previousSibling, None) - - def test_previous_sibling(self): - self.assertEquals(self.end.previousSibling['id'], '3') - self.assertEquals(self.end.previousSibling.previousSibling['id'], '2') - - # Note the difference between previousSibling and previous. - self.assertEquals(self.end.previous['id'], '3.1') - - def test_previous_sibling_may_not_exist(self): - self.assertEquals(self.tree.html.previousSibling, None) - - nested_span = self.tree.find(id="1.1") - self.assertEquals(nested_span.previousSibling, None) - - first_span = self.tree.find(id="1") - self.assertEquals(first_span.previousSibling, None) - - def test_find_previous_sibling(self): - self.assertEquals(self.end.findPreviousSibling('span')['id'], '3') - - def test_previous_siblings(self): - self.assertSelectsIDs(self.end.findPreviousSiblings("span"), - ['3', '2', '1']) - - self.assertSelectsIDs(self.end.findPreviousSiblings(id='1'), ['1']) - - def test_previous_sibling_for_text_element(self): - soup = self.soup("Foo<b>bar</b>baz") - start = soup.find(text="baz") - self.assertEquals(start.previousSibling.name, 'b') - self.assertEquals(start.previousSibling.previousSibling, 'Foo') - - self.assertSelects(start.findPreviousSiblings('b'), ['bar']) - self.assertEquals(start.findPreviousSibling(text="Foo"), "Foo") - self.assertEquals(start.findPreviousSibling(text="nonesuch"), None) - - -class TestTreeModification(SoupTest): - - def test_attribute_modification(self): - soup = self.soup('<a id="1"></a>') - soup.a['id'] = 2 - self.assertEqual(soup.decode(), self.document_for('<a id="2"></a>')) - del(soup.a['id']) - self.assertEqual(soup.decode(), self.document_for('<a></a>')) - soup.a['id2'] = 'foo' - self.assertEqual(soup.decode(), self.document_for('<a id2="foo"></a>')) - - def test_new_tag_creation(self): - builder = BeautifulSoup.default_builder() - soup = BeautifulSoup("", builder=builder) - a = Tag(soup, builder, 'a') - ol = Tag(soup, builder, 'ol') - a['href'] = 'http://foo.com/' - soup.insert(0, a) - soup.insert(1, ol) - self.assertEqual( - soup.decode(), '<a href="http://foo.com/"></a><ol></ol>') - - def test_append_to_contents_moves_tag(self): - doc = """<p id="1">Don't leave me <b>here</b>.</p> - <p id="2">Don\'t leave!</p>""" - soup = self.soup(doc) - second_para = soup.find(id='2') - bold = soup.b - - # Move the <b> tag to the end of the second paragraph. - soup.find(id='2').append(soup.b) - - # The <b> tag is now a child of the second paragraph. - self.assertEqual(bold.parent, second_para) - - self.assertEqual( - soup.decode(), self.document_for( - '<p id="1">Don\'t leave me .</p>\n' - '<p id="2">Don\'t leave!<b>here</b></p>')) - - def test_replace_tag_with_itself(self): - text = "<a><b></b><c>Foo<d></d></c></a><a><e></e></a>" - soup = BeautifulSoup(text) - c = soup.c - soup.c.replaceWith(c) - self.assertEquals(soup.decode(), self.document_for(text)) - - def test_replace_final_node(self): - soup = self.soup("<b>Argh!</b>") - soup.find(text="Argh!").replaceWith("Hooray!") - new_text = soup.find(text="Hooray!") - b = soup.b - self.assertEqual(new_text.previous, b) - self.assertEqual(new_text.parent, b) - self.assertEqual(new_text.previous.next, new_text) - self.assertEqual(new_text.next, None) - - def test_consecutive_text_nodes(self): - # A builder should never create two consecutive text nodes, - # but if you insert one next to another, Beautiful Soup will - # handle it correctly. - soup = self.soup("<a><b>Argh!</b><c></c></a>") - soup.b.insert(1, "Hooray!") - - self.assertEqual( - soup.decode(), self.document_for( - "<a><b>Argh!Hooray!</b><c></c></a>")) - - new_text = soup.find(text="Hooray!") - self.assertEqual(new_text.previous, "Argh!") - self.assertEqual(new_text.previous.next, new_text) - - self.assertEqual(new_text.previousSibling, "Argh!") - self.assertEqual(new_text.previousSibling.nextSibling, new_text) - - self.assertEqual(new_text.nextSibling, None) - self.assertEqual(new_text.next, soup.c) - - - def test_insert_tag(self): - builder = self.default_builder - soup = BeautifulSoup( - "<a><b>Find</b><c>lady!</c><d></d></a>", builder=builder) - magic_tag = Tag(soup, builder, 'magictag') - magic_tag.insert(0, "the") - soup.a.insert(1, magic_tag) - - self.assertEqual( - soup.decode(), self.document_for( - "<a><b>Find</b><magictag>the</magictag><c>lady!</c><d></d></a>")) - - # Make sure all the relationships are hooked up correctly. - b_tag = soup.b - self.assertEqual(b_tag.nextSibling, magic_tag) - self.assertEqual(magic_tag.previousSibling, b_tag) - - find = b_tag.find(text="Find") - self.assertEqual(find.next, magic_tag) - self.assertEqual(magic_tag.previous, find) - - c_tag = soup.c - self.assertEqual(magic_tag.nextSibling, c_tag) - self.assertEqual(c_tag.previousSibling, magic_tag) - - the = magic_tag.find(text="the") - self.assertEqual(the.parent, magic_tag) - self.assertEqual(the.next, c_tag) - self.assertEqual(c_tag.previous, the) - - def test_replace_with(self): - soup = self.soup( - "<p>There's <b>no</b> business like <b>show</b> business</p>") - no, show = soup.findAll('b') - show.replaceWith(no) - self.assertEquals( - soup.decode(), - self.document_for( - "<p>There's business like <b>no</b> business</p>")) - - self.assertEquals(show.parent, None) - self.assertEquals(no.parent, soup.p) - self.assertEquals(no.next, "no") - self.assertEquals(no.nextSibling, " business") - - def test_nested_tag_replace_with(self): - soup = BeautifulSoup( - """<a>We<b>reserve<c>the</c><d>right</d></b></a><e>to<f>refuse</f><g>service</g></e>""") - - # Replace the entire <b> tag and its contents ("reserve the - # right") with the <f> tag ("refuse"). - remove_tag = soup.b - move_tag = soup.f - remove_tag.replaceWith(move_tag) - - self.assertEqual( - soup.decode(), self.document_for( - "<a>We<f>refuse</f></a><e>to<g>service</g></e>")) - - # The <b> tag is now an orphan. - self.assertEqual(remove_tag.parent, None) - self.assertEqual(remove_tag.find(text="right").next, None) - self.assertEqual(remove_tag.previous, None) - self.assertEqual(remove_tag.nextSibling, None) - self.assertEqual(remove_tag.previousSibling, None) - - # The <f> tag is now connected to the <a> tag. - self.assertEqual(move_tag.parent, soup.a) - self.assertEqual(move_tag.previous, "We") - self.assertEqual(move_tag.next.next, soup.e) - self.assertEqual(move_tag.nextSibling, None) - - # The gap where the <f> tag used to be has been mended, and - # the word "to" is now connected to the <g> tag. - to_text = soup.find(text="to") - g_tag = soup.g - self.assertEqual(to_text.next, g_tag) - self.assertEqual(to_text.nextSibling, g_tag) - self.assertEqual(g_tag.previous, to_text) - self.assertEqual(g_tag.previousSibling, to_text) - - def test_extract(self): - soup = self.soup( - '<html><body>Some content. <div id="nav">Nav crap</div> More content.</body></html>') - - self.assertEqual(len(soup.body.contents), 3) - extracted = soup.find(id="nav").extract() - - self.assertEqual( - soup.decode(), "<html><body>Some content. More content.</body></html>") - self.assertEqual(extracted.decode(), '<div id="nav">Nav crap</div>') - - # The extracted tag is now an orphan. - self.assertEqual(len(soup.body.contents), 2) - self.assertEqual(extracted.parent, None) - self.assertEqual(extracted.previous, None) - self.assertEqual(extracted.next.next, None) - - # The gap where the extracted tag used to be has been mended. - content_1 = soup.find(text="Some content. ") - content_2 = soup.find(text=" More content.") - self.assertEquals(content_1.next, content_2) - self.assertEquals(content_1.nextSibling, content_2) - self.assertEquals(content_2.previous, content_1) - self.assertEquals(content_2.previousSibling, content_1) - - -class TestElementObjects(SoupTest): - """Test various features of element objects.""" - - def test_len(self): - """The length of an element is its number of children.""" - soup = self.soup("<top>1<b>2</b>3</top>") - - # The BeautifulSoup object itself contains one element: the - # <top> tag. - self.assertEquals(len(soup.contents), 1) - self.assertEquals(len(soup), 1) - - # The <top> tag contains three elements: the text node "1", the - # <b> tag, and the text node "3". - self.assertEquals(len(soup.top), 3) - self.assertEquals(len(soup.top.contents), 3) - - def test_member_access_invokes_find(self): - """Accessing a Python member .foo or .fooTag invokes find('foo')""" - soup = self.soup('<b><i></i></b>') - self.assertEqual(soup.b, soup.find('b')) - self.assertEqual(soup.bTag, soup.find('b')) - self.assertEqual(soup.b.i, soup.find('b').find('i')) - self.assertEqual(soup.bTag.iTag, soup.find('b').find('i')) - self.assertEqual(soup.a, None) - self.assertEqual(soup.aTag, None) - - def test_has_key(self): - """has_key() checks for the presence of an attribute.""" - soup = self.soup("<foo attr='bar'>") - self.assertTrue(soup.foo.has_key('attr')) - self.assertFalse(soup.foo.has_key('attr2')) - - def test_string(self): - # A tag that contains only a text node makes that node - # available as .string. - soup = self.soup("<b>foo</b>") - self.assertEquals(soup.b.string, 'foo') - - def test_lack_of_string(self): - """Only a tag containing a single text node has a .string.""" - soup = self.soup("<b>f<i>e</i>o</b>") - self.assertFalse(soup.b.string) - - soup = self.soup("<b></b>") - self.assertFalse(soup.b.string) |