diff options
| author | Leonard Richardson <leonardr@segfault.org> | 2023-01-25 14:36:30 -0500 |
|---|---|---|
| committer | Leonard Richardson <leonardr@segfault.org> | 2023-01-25 14:36:30 -0500 |
| commit | e4f967730af7d5ea59f6a05d03c779df16a24bd3 (patch) | |
| tree | 02af82ae27b992e4e863db21fd0e21ebe8584819 | |
| parent | d84821852a32d1314a8fcb5fa4180e3beecea39c (diff) | |
Removed very old tests that were imported as part of the bzr import but not removed.
| -rw-r--r-- | tests/test_builder_registry.py | 121 | ||||
| -rw-r--r-- | tests/test_html5lib.py | 226 | ||||
| -rw-r--r-- | tests/test_htmlparser.py | 126 | ||||
| -rw-r--r-- | tests/test_lxml.py | 586 | ||||
| -rw-r--r-- | tests/test_soup.py | 130 | ||||
| -rw-r--r-- | tests/test_tree.py | 982 |
6 files changed, 0 insertions, 2171 deletions
diff --git a/tests/test_builder_registry.py b/tests/test_builder_registry.py deleted file mode 100644 index 655cd06..0000000 --- a/tests/test_builder_registry.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Tests of the builder registry.""" - -import unittest - -from bs4 import BeautifulSoup -from bs4.builder import ( - builder_registry as registry, - LXMLTreeBuilderForXML, - LXMLTreeBuilder, - TreeBuilderRegistry, - HTML5TreeBuilder, - HTMLParserTreeBuilder, -) - - - -class BuiltInRegistryTest(unittest.TestCase): - """Test the built-in registry with the default builders registered.""" - - def test_combination(self): - self.assertEquals(registry.lookup('fast', 'html'), - LXMLTreeBuilder) - self.assertEquals(registry.lookup('permissive', 'xml'), - LXMLTreeBuilderForXML) - self.assertEquals(registry.lookup('strict', 'html'), - HTMLParserTreeBuilder) - self.assertEquals(registry.lookup('permissive', 'html'), - HTML5TreeBuilder) - - def test_lookup_by_markup_type(self): - self.assertEquals(registry.lookup('html'), HTML5TreeBuilder) - self.assertEquals(registry.lookup('xml'), LXMLTreeBuilderForXML) - - def test_named_library(self): - self.assertEquals(registry.lookup('lxml', 'xml'), - LXMLTreeBuilderForXML) - self.assertEquals(registry.lookup('lxml', 'html'), - LXMLTreeBuilder) - self.assertEquals(registry.lookup('html5lib'), - HTML5TreeBuilder) - - self.assertEquals(registry.lookup('html.parser'), - HTMLParserTreeBuilder) - - def test_unimplemented_combinations(self): - self.assertEquals(registry.lookup('fast', 'permissive', 'html'), - None) - - def test_beautifulsoup_constructor_does_lookup(self): - # You can pass in a string. - BeautifulSoup("", features="html") - # Or a list of strings. - BeautifulSoup("", features=["html", "permissive"]) - - # You'll get an exception if BS can't find an appropriate - # builder. - self.assertRaises(ValueError, BeautifulSoup, - "", features="no-such-feature") - -class RegistryTest(unittest.TestCase): - """Test the TreeBuilderRegistry class in general.""" - - def setUp(self): - self.registry = TreeBuilderRegistry() - - def builder_for_features(self, *feature_list): - cls = type('Builder_' + '_'.join(feature_list), - (object,), {'features' : feature_list}) - - self.registry.register(cls) - return cls - - def test_register_with_no_features(self): - builder = self.builder_for_features() - - # Since the builder advertises no features, you can't find it - # by looking up features. - self.assertEquals(self.registry.lookup('foo'), None) - - # But you can find it by doing a lookup with no features, if - # this happens to be the only registered builder. - self.assertEquals(self.registry.lookup(), builder) - - def test_register_with_features_makes_lookup_succeed(self): - builder = self.builder_for_features('foo', 'bar') - self.assertEquals(self.registry.lookup('foo'), builder) - self.assertEquals(self.registry.lookup('bar'), builder) - - def test_lookup_fails_when_no_builder_implements_feature(self): - builder = self.builder_for_features('foo', 'bar') - self.assertEquals(self.registry.lookup('baz'), None) - - def test_lookup_gets_most_recent_registration_when_no_feature_specified(self): - builder1 = self.builder_for_features('foo') - builder2 = self.builder_for_features('bar') - self.assertEquals(self.registry.lookup(), builder2) - - def test_lookup_fails_when_no_tree_builders_registered(self): - self.assertEquals(self.registry.lookup(), None) - - def test_lookup_gets_most_recent_builder_supporting_all_features(self): - has_one = self.builder_for_features('foo') - has_the_other = self.builder_for_features('bar') - has_both_early = self.builder_for_features('foo', 'bar', 'baz') - has_both_late = self.builder_for_features('foo', 'bar', 'quux') - lacks_one = self.builder_for_features('bar') - has_the_other = self.builder_for_features('foo') - - # There are two builders featuring 'foo' and 'bar', but - # the one that also features 'quux' was registered later. - self.assertEquals(self.registry.lookup('foo', 'bar'), - has_both_late) - - # There is only one builder featuring 'foo', 'bar', and 'baz'. - self.assertEquals(self.registry.lookup('foo', 'bar', 'baz'), - has_both_early) - - def test_lookup_fails_when_cannot_reconcile_requested_features(self): - builder1 = self.builder_for_features('foo', 'bar') - builder2 = self.builder_for_features('foo', 'baz') - self.assertEquals(self.registry.lookup('bar', 'baz'), None) diff --git a/tests/test_html5lib.py b/tests/test_html5lib.py deleted file mode 100644 index ac99832..0000000 --- a/tests/test_html5lib.py +++ /dev/null @@ -1,226 +0,0 @@ -from bs4.builder import HTML5TreeBuilder -from bs4.element import Comment, SoupStrainer -from test_lxml import ( - TestLXMLBuilder, - TestLXMLBuilderInvalidMarkup, - TestLXMLBuilderEncodingConversion, - ) - -class TestHTML5Builder(TestLXMLBuilder): - """See `BuilderSmokeTest`.""" - - @property - def default_builder(self): - return HTML5TreeBuilder() - - def test_soupstrainer(self): - # The html5lib tree builder does not support SoupStrainers. - strainer = SoupStrainer("b") - markup = "<p>A <b>bold</b> statement.</p>" - soup = self.soup(markup, - parse_only=strainer) - self.assertEquals( - soup.decode(), self.document_for(markup)) - - def test_bare_string(self): - # A bare string is turned into some kind of HTML document or - # fragment recognizable as the original string. - # - # In this case, lxml puts a <p> tag around the bare string. - self.assertSoupEquals( - "A bare string", "A bare string") - - def test_correctly_nested_tables(self): - markup = ('<table id="1">' - '<tr>' - "<td>Here's another table:" - '<table id="2">' - '<tr><td>foo</td></tr>' - '</table></td>') - - self.assertSoupEquals( - markup, - '<table id="1"><tbody><tr><td>Here\'s another table:' - '<table id="2"><tbody><tr><td>foo</td></tr></tbody></table>' - '</td></tr></tbody></table>') - - self.assertSoupEquals( - "<table><thead><tr><td>Foo</td></tr></thead>" - "<tbody><tr><td>Bar</td></tr></tbody>" - "<tfoot><tr><td>Baz</td></tr></tfoot></table>") - - def test_literal_in_textarea(self): - markup = '<textarea>Junk like <b> tags and <&<&</textarea>' - soup = self.soup(markup) - self.assertEquals( - soup.textarea.contents, ["Junk like <b> tags and <&<&"]) - - def test_collapsed_whitespace(self): - """Whitespace is preserved even in tags that don't require it.""" - self.assertSoupEquals("<p> </p>") - self.assertSoupEquals("<b> </b>") - - def test_cdata_where_its_ok(self): - # In html5lib 0.9.0, all CDATA sections are converted into - # comments. In a later version (unreleased as of this - # writing), CDATA sections in tags like <svg> and <math> will - # be preserved. BUT, I'm not sure how Beautiful Soup needs to - # adjust to transform this preservation into the construction - # of a BS CData object. - markup = "<svg><![CDATA[foobar]]>" - - # Eventually we should be able to do a find(text="foobar") and - # get a CData object. - self.assertSoupEquals(markup, "<svg><!--[CDATA[foobar]]--></svg>") - - -class TestHTML5BuilderInvalidMarkup(TestLXMLBuilderInvalidMarkup): - """See `BuilderInvalidMarkupSmokeTest`.""" - - @property - def default_builder(self): - return HTML5TreeBuilder() - - def test_unclosed_block_level_elements(self): - # The unclosed <b> tag is closed so that the block-level tag - # can be closed, and another <b> tag is inserted after the - # next block-level tag begins. - self.assertSoupEquals( - '<blockquote><p><b>Foo</blockquote><p>Bar', - '<blockquote><p><b>Foo</b></p></blockquote><p><b>Bar</b></p>') - - def test_table_containing_bare_markup(self): - # Markup should be in table cells, not directly in the table. - self.assertSoupEquals("<table><div>Foo</div></table>", - "<div>Foo</div><table></table>") - - def test_incorrectly_nested_tables(self): - self.assertSoupEquals( - '<table><tr><table><tr id="nested">', - ('<table><tbody><tr></tr></tbody></table>' - '<table><tbody><tr id="nested"></tr></tbody></table>')) - - def test_empty_element_tag_with_contents(self): - self.assertSoupEquals("<br>foo</br>", "<br />foo<br />") - - def test_doctype_in_body(self): - markup = "<p>one<!DOCTYPE foobar>two</p>" - self.assertSoupEquals(markup, "<p>onetwo</p>") - - def test_cdata_where_it_doesnt_belong(self): - # Random CDATA sections are converted into comments. - markup = "<div><![CDATA[foo]]>" - soup = self.soup(markup) - data = soup.find(text="[CDATA[foo]]") - self.assertEquals(data.__class__, Comment) - - def test_nonsensical_declaration(self): - # Declarations that don't make any sense are turned into comments. - soup = self.soup('<! Foo = -8><p>a</p>') - self.assertEquals(str(soup), - ("<!-- Foo = -8-->" - "<html><head></head><body><p>a</p></body></html>")) - - soup = self.soup('<p>a</p><! Foo = -8>') - self.assertEquals(str(soup), - ("<html><head></head><body><p>a</p>" - "<!-- Foo = -8--></body></html>")) - - def test_whitespace_in_doctype(self): - # A declaration that has extra whitespace is turned into a comment. - soup = self.soup(( - '<! DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">' - '<p>foo</p>')) - self.assertEquals( - str(soup), - ('<!-- DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"-->' - '<html><head></head><body><p>foo</p></body></html>')) - - def test_incomplete_declaration(self): - # An incomplete declaration is treated as a comment. - markup = 'a<!b <p>c' - self.assertSoupEquals(markup, "a<!--b <p-->c") - - # Let's spell that out a little more explicitly. - soup = self.soup(markup) - str1, comment, str2 = soup.body.contents - self.assertEquals(str1, 'a') - self.assertEquals(comment.__class__, Comment) - self.assertEquals(comment, 'b <p') - self.assertEquals(str2, 'c') - - def test_document_starts_with_bogus_declaration(self): - soup = self.soup('<! Foo >a') - # 'Foo' becomes a comment that appears before the HTML. - comment = soup.contents[0] - self.assertTrue(isinstance(comment, Comment)) - self.assertEquals(comment, 'Foo') - - self.assertEquals(self.find(text="a") == "a") - - def test_attribute_value_was_closed_by_subsequent_tag(self): - markup = """<a href="foo</a>, </a><a href="bar">baz</a>""" - soup = self.soup(markup) - # The string between the first and second quotes was interpreted - # as the value of the 'href' attribute. - self.assertEquals(soup.a['href'], 'foo</a>, </a><a href=') - - #The string after the second quote (bar"), was treated as an - #empty attribute called bar". - self.assertEquals(soup.a['bar"'], '') - self.assertEquals(soup.a.string, "baz") - - def test_document_starts_with_bogus_declaration(self): - soup = self.soup('<! Foo ><p>a</p>') - # The declaration becomes a comment. - comment = soup.contents[0] - self.assertTrue(isinstance(comment, Comment)) - self.assertEquals(comment, ' Foo ') - self.assertEquals(soup.p.string, 'a') - - def test_document_ends_with_incomplete_declaration(self): - soup = self.soup('<p>a<!b') - # This becomes a string 'a'. The incomplete declaration is ignored. - # Compare html5lib, which turns it into a comment. - s, comment = soup.p.contents - self.assertEquals(s, 'a') - self.assertTrue(isinstance(comment, Comment)) - self.assertEquals(comment, 'b') - - def test_entity_was_not_finished(self): - soup = self.soup("<p><Hello>") - # Compare html5lib, which completes the entity. - self.assertEquals(soup.p.string, "<Hello>") - - def test_nonexistent_entity(self): - soup = self.soup("<p>foo&#bar;baz</p>") - self.assertEquals(soup.p.string, "foo&#bar;baz") - - # Compare a real entity. - soup = self.soup("<p>foodbaz</p>") - self.assertEquals(soup.p.string, "foodbaz") - - def test_entity_out_of_range(self): - # An entity that's out of range will be converted to - # REPLACEMENT CHARACTER. - soup = self.soup("<p>�</p>") - self.assertEquals(soup.p.string, u"\N{REPLACEMENT CHARACTER}") - - soup = self.soup("<p>�</p>") - self.assertEquals(soup.p.string, u"\N{REPLACEMENT CHARACTER}") - - -class TestHTML5LibEncodingConversion(TestLXMLBuilderEncodingConversion): - @property - def default_builder(self): - return HTML5TreeBuilder() - - def test_real_hebrew_document(self): - # A real-world test to make sure we can convert ISO-8859-9 (a - # Hebrew encoding) to UTF-8. - soup = self.soup(self.HEBREW_DOCUMENT, - from_encoding="iso-8859-8") - self.assertEquals(soup.original_encoding, 'iso8859-8') - self.assertEquals( - soup.encode('utf-8'), - self.HEBREW_DOCUMENT.decode("iso-8859-8").encode("utf-8")) diff --git a/tests/test_htmlparser.py b/tests/test_htmlparser.py deleted file mode 100644 index c8a446e..0000000 --- a/tests/test_htmlparser.py +++ /dev/null @@ -1,126 +0,0 @@ -from HTMLParser import HTMLParseError -from bs4.builder import HTMLParserTreeBuilder -from bs4.element import CData -from test_lxml import ( - TestLXMLBuilder, - TestLXMLBuilderEncodingConversion, - TestLXMLBuilderInvalidMarkup, - ) - -class TestHTMLParserTreeBuilder(TestLXMLBuilder): - """See `BuilderSmokeTest`.""" - - @property - def default_builder(self): - return HTMLParserTreeBuilder() - - def test_bare_string(self): - # A bare string is turned into some kind of HTML document or - # fragment recognizable as the original string. - # - # HTMLParser does not modify the bare string at all. - self.assertSoupEquals("A bare string") - - def test_cdata_where_its_ok(self): - # HTMLParser recognizes CDATA sections and passes them through. - markup = "<svg><![CDATA[foobar]]></svg>" - self.assertSoupEquals(markup) - soup = self.soup(markup) - string = soup.svg.string - self.assertEquals(string, "foobar") - self.assertTrue(isinstance(string, CData)) - - # These are tests that could be 'fixed' by improving the - # HTMLParserTreeBuilder, but I don't think it's worth it. Users - # will have fewer headaches if they use one of the other tree - # builders. - - def test_empty_element(self): - # HTML's empty-element tags are not recognized as such - # unless they are presented as empty-element tags. - self.assertSoupEquals( - "<p>A <meta> tag</p>", "<p>A <meta> tag</meta></p>") - - self.assertSoupEquals( - "<p>Foo<br/>bar</p>", "<p>Foo<br />bar</p>") - - def test_entities_in_attribute_values_converted_during_parsing(self): - - # The numeric entity isn't recognized without the closing - # semicolon. - text = '<x t="piñata">' - expected = u"pi\N{LATIN SMALL LETTER N WITH TILDE}ata" - soup = self.soup(text) - self.assertEquals(soup.x['t'], "piñata") - - text = '<x t="piñata">' - expected = u"pi\N{LATIN SMALL LETTER N WITH TILDE}ata" - soup = self.soup(text) - self.assertEquals(soup.x['t'], u"pi\xf1ata") - - text = '<x t="piñata">' - soup = self.soup(text) - self.assertEquals(soup.x['t'], expected) - - text = '<x t="sacré bleu">' - soup = self.soup(text) - self.assertEquals( - soup.x['t'], - u"sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu") - - # This can cause valid HTML to become invalid. - valid_url = '<a href="http://example.org?a=1&b=2;3">foo</a>' - soup = self.soup(valid_url) - self.assertEquals(soup.a['href'], "http://example.org?a=1&b=2;3") - - # I think it would be very difficult to 'fix' these tests, judging - # from my experience with previous versions of Beautiful Soup. - def test_naked_ampersands(self): - # Ampersands are treated as entities. - text = "<p>AT&T</p>" - soup = self.soup(text) - self.assertEquals(soup.p.string, "AT&T;") - - def test_literal_in_textarea(self): - # Anything inside a <textarea> is supposed to be treated as - # the literal value of the field, (XXX citation - # needed). html5lib does this correctly. But, HTMLParser does its - # best to parse the contents of a <textarea> as HTML. - text = '<textarea>Junk like <b> tags and <&<&</textarea>' - soup = self.soup(text) - self.assertEquals(len(soup.textarea.contents), 2) - self.assertEquals(soup.textarea.contents[0], u"Junk like ") - self.assertEquals(soup.textarea.contents[1].name, 'b') - self.assertEquals(soup.textarea.b.string, u" tags and <&<&") - - def test_literal_in_script(self): - # The contents of a <script> tag are supposed to be treated as - # a literal string, even if that string contains HTML. But - # HTMLParser attempts to parse some of the HTML, causing much - # pain. - javascript = 'if (i < 2) { alert("<b>foo</b>"); }' - soup = self.soup('<script>%s</script>' % javascript) - self.assertEquals(soup.script.contents, - ['if (i < 2) { alert("<b>foo', - '"); }']) - - # Namespaced doctypes cause an HTMLParseError - def test_namespaced_system_doctype(self): - self.assertRaises(HTMLParseError, self._test_doctype, - 'xsl:stylesheet SYSTEM "htmlent.dtd"') - - def test_namespaced_public_doctype(self): - self.assertRaises(HTMLParseError, self._test_doctype, - 'xsl:stylesheet PUBLIC "htmlent.dtd"') - - -class TestHTMLParserTreeBuilderInvalidMarkup(TestLXMLBuilderInvalidMarkup): - # Oddly enough, HTMLParser seems to handle invalid markup exactly - # the same as lxml. - pass - - -class TestHTMLParserTreeBuilderEncodingConversion( - TestLXMLBuilderEncodingConversion): - # Re-run the lxml tests for HTMLParser - pass diff --git a/tests/test_lxml.py b/tests/test_lxml.py deleted file mode 100644 index 7ce33e8..0000000 --- a/tests/test_lxml.py +++ /dev/null @@ -1,586 +0,0 @@ -"""Tests to ensure that the lxml tree builder generates good trees.""" - -import re - -from bs4 import BeautifulSoup -from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML -from bs4.element import Comment, Doctype, SoupStrainer -from bs4.testing import SoupTest - - -class TestLXMLBuilder(SoupTest): - """A smoke test for the LXML tree builder. - - Subclass this to test some other HTML tree builder. Subclasses of - this test ensure that all of Beautiful Soup's tree builders - generate more or less the same trees. - - It's okay for trees to differ--just override the appropriate test - method to demonstrate how one tree builder differs from the LXML - builder. But in general, all HTML tree builders should generate - trees that make most of these tests pass. - """ - - def test_bare_string(self): - # A bare string is turned into some kind of HTML document or - # fragment recognizable as the original string. - # - # In this case, lxml puts a <p> tag around the bare string. - self.assertSoupEquals( - "A bare string", "<p>A bare string</p>") - - def test_mixed_case_tags(self): - # Mixed-case tags are folded to lowercase. - self.assertSoupEquals( - "<a><B><Cd><EFG></efg></CD></b></A>", - "<a><b><cd><efg></efg></cd></b></a>") - - def test_empty_element(self): - # HTML's empty-element tags are recognized as such. - self.assertSoupEquals( - "<p>A <meta> tag</p>", "<p>A <meta /> tag</p>") - - self.assertSoupEquals( - "<p>Foo<br/>bar</p>", "<p>Foo<br />bar</p>") - - def test_empty_tag_thats_not_an_empty_element_tag(self): - # A tag that is empty but not an HTML empty-element tag - # is not presented as an empty-element tag. - self.assertSoupEquals("<p>", "<p></p>") - - def test_comment(self): - # Comments are represented as Comment objects. - markup = "<p>foo<!--foobar-->baz</p>" - self.assertSoupEquals(markup) - - soup = self.soup(markup) - comment = soup.find(text="foobar") - self.assertEquals(comment.__class__, Comment) - - def test_nested_inline_elements(self): - # Inline tags can be nested indefinitely. - b_tag = "<b>Inside a B tag</b>" - self.assertSoupEquals(b_tag) - - nested_b_tag = "<p>A <i>nested <b>tag</b></i></p>" - self.assertSoupEquals(nested_b_tag) - - double_nested_b_tag = "<p>A <a>doubly <i>nested <b>tag</b></i></a></p>" - self.assertSoupEquals(nested_b_tag) - - def test_nested_block_level_elements(self): - soup = self.soup('<blockquote><p><b>Foo</b></p></blockquote>') - blockquote = soup.blockquote - self.assertEqual(blockquote.p.b.string, 'Foo') - self.assertEqual(blockquote.b.string, 'Foo') - - # This is a <table> tag containing another <table> tag in one of its - # cells. - TABLE_MARKUP_1 = ('<table id="1">' - '<tr>' - "<td>Here's another table:" - '<table id="2">' - '<tr><td>foo</td></tr>' - '</table></td>') - - def test_correctly_nested_tables(self): - markup = ('<table id="1">' - '<tr>' - "<td>Here's another table:" - '<table id="2">' - '<tr><td>foo</td></tr>' - '</table></td>') - - self.assertSoupEquals( - markup, - '<table id="1"><tr><td>Here\'s another table:' - '<table id="2"><tr><td>foo</td></tr></table>' - '</td></tr></table>') - - self.assertSoupEquals( - "<table><thead><tr><td>Foo</td></tr></thead>" - "<tbody><tr><td>Bar</td></tr></tbody>" - "<tfoot><tr><td>Baz</td></tr></tfoot></table>") - - def test_collapsed_whitespace(self): - """In most tags, whitespace is collapsed.""" - self.assertSoupEquals("<p> </p>", "<p> </p>") - - def test_preserved_whitespace_in_pre_and_textarea(self): - """In <pre> and <textarea> tags, whitespace is preserved.""" - self.assertSoupEquals("<pre> </pre>") - self.assertSoupEquals("<textarea> woo </textarea>") - - def test_single_quote_attribute_values_become_double_quotes(self): - self.assertSoupEquals("<foo attr='bar'></foo>", - '<foo attr="bar"></foo>') - - def test_attribute_values_with_nested_quotes_are_left_alone(self): - text = """<foo attr='bar "brawls" happen'>a</foo>""" - self.assertSoupEquals(text) - - def test_attribute_values_with_double_nested_quotes_get_quoted(self): - text = """<foo attr='bar "brawls" happen'>a</foo>""" - soup = self.soup(text) - soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"' - self.assertSoupEquals( - soup.foo.decode(), - """<foo attr="Brawls happen at "Bob\'s Bar"">a</foo>""") - - def test_ampersand_in_attribute_value_gets_quoted(self): - self.assertSoupEquals('<this is="really messed up & stuff"></this>', - '<this is="really messed up & stuff"></this>') - - def test_literal_in_textarea(self): - # Anything inside a <textarea> is supposed to be treated as - # the literal value of the field, (XXX citation - # needed). html5lib does this correctly. But, lxml does its - # best to parse the contents of a <textarea> as HTML. - text = '<textarea>Junk like <b> tags and <&<&</textarea>' - soup = self.soup(text) - self.assertEquals(len(soup.textarea.contents), 2) - self.assertEquals(soup.textarea.contents[0], u"Junk like ") - self.assertEquals(soup.textarea.contents[1].name, 'b') - self.assertEquals(soup.textarea.b.string, u" tags and ") - - def test_literal_in_script(self): - # The contents of a <script> tag are treated as a literal string, - # even if that string contains HTML. - javascript = 'if (i < 2) { alert("<b>foo</b>"); }' - soup = self.soup('<script>%s</script>' % javascript) - self.assertEquals(soup.script.string, javascript) - - def test_naked_ampersands(self): - # Ampersands are left alone. - text = "<p>AT&T</p>" - soup = self.soup(text) - self.assertEquals(soup.p.string, "AT&T") - - # Even if they're in attribute values. - invalid_url = '<a href="http://example.org?a=1&b=2;3">foo</a>' - soup = self.soup(invalid_url) - self.assertEquals(soup.a['href'], "http://example.org?a=1&b=2;3") - - def test_entities_in_strings_converted_during_parsing(self): - # Both XML and HTML entities are converted to Unicode characters - # during parsing. - text = "<p><<sacré bleu!>></p>" - expected = u"<p><<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></p>" - self.assertSoupEquals(text, expected) - - def test_smart_quotes_converted_on_the_way_in(self): - # Microsoft smart quotes are converted to Unicode characters during - # parsing. - quote = "<p>\x91Foo\x92</p>" - soup = self.soup(quote) - self.assertEquals( - soup.p.string, - u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}") - - def test_non_breaking_spaces_converted_on_the_way_in(self): - soup = self.soup("<a> </a>") - self.assertEquals(soup.a.string, u"\N{NO-BREAK SPACE}" * 2) - - def test_cdata_where_its_ok(self): - # lxml strips CDATA sections, no matter where they occur. - markup = "<svg><![CDATA[foobar]]>" - self.assertSoupEquals(markup, "<svg></svg>") - - def _test_doctype(self, doctype_fragment): - """Run a battery of assertions on a given doctype string.""" - doctype_str = '<!DOCTYPE %s>' % doctype_fragment - markup = doctype_str + '<p>foo</p>' - soup = self.soup(markup) - doctype = soup.contents[0] - self.assertEquals(doctype.__class__, Doctype) - self.assertEquals(doctype, doctype_fragment) - self.assertEquals(str(soup)[:len(doctype_str)], doctype_str) - - # Make sure that the doctype was correctly associated with the - # parse tree and that the rest of the document parsed. - self.assertEquals(soup.p.contents[0], 'foo') - - def test_doctype(self): - # Test a normal HTML doctype you'll commonly see in a real document. - self._test_doctype( - 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"') - - def test_namespaced_system_doctype(self): - # Test a namespaced doctype with a system id. - self._test_doctype('xsl:stylesheet SYSTEM "htmlent.dtd"') - - def test_namespaced_public_doctype(self): - # Test a namespaced doctype with a public id. - self._test_doctype('xsl:stylesheet PUBLIC "htmlent.dtd"') - - def test_real_iso_latin_document(self): - # Smoke test of interrelated functionality, using an - # easy-to-understand document. - - # Here it is in Unicode. Note that it claims to be in ISO-Latin-1. - unicode_html = u'<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type" /></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>' - - # That's because we're going to encode it into ISO-Latin-1, and use - # that to test. - iso_latin_html = unicode_html.encode("iso-8859-1") - - # Parse the ISO-Latin-1 HTML. - soup = self.soup(iso_latin_html) - # Encode it to UTF-8. - result = soup.encode("utf-8") - - # What do we expect the result to look like? Well, it would - # look like unicode_html, except that the META tag would say - # UTF-8 instead of ISO-Latin-1. - expected = unicode_html.replace("ISO-Latin-1", "utf-8") - - # And, of course, it would be in UTF-8, not Unicode. - expected = expected.encode("utf-8") - - # Ta-da! - self.assertEquals(result, expected) - - def test_real_shift_jis_document(self): - # Smoke test to make sure the parser can handle a document in - # Shift-JIS encoding, without choking. - shift_jis_html = ( - '<html><head></head><body><pre>' - '\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f' - '\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c' - '\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B' - '</pre></body></html>') - unicode_html = shift_jis_html.decode("shift-jis") - soup = self.soup(shift_jis_html) - - # Make sure the parse tree is correctly encoded to various - # encodings. - self.assertEquals(soup.encode("utf-8"), unicode_html.encode("utf-8")) - self.assertEquals(soup.encode("euc_jp"), unicode_html.encode("euc_jp")) - - # Tests below this line need work. - - def test_meta_tag_reflects_current_encoding(self): - # Here's the <meta> tag saying that a document is - # encoded in Shift-JIS. - meta_tag = ('<meta content="text/html; charset=x-sjis" ' - 'http-equiv="Content-type" />') - - # Here's a document incorporating that meta tag. - shift_jis_html = ( - '<html><head>\n%s\n' - '<meta http-equiv="Content-language" content="ja" />' - '</head><body>Shift-JIS markup goes here.') % meta_tag - soup = self.soup(shift_jis_html) - - # Parse the document, and the charset is replaced with a - # generic value. - parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'}) - self.assertEquals(parsed_meta['content'], - 'text/html; charset=%SOUP-ENCODING%') - self.assertEquals(parsed_meta.contains_substitutions, True) - - # For the rest of the story, see TestSubstitutions in - # test_tree.py. - - def test_entities_converted_on_the_way_out(self): - text = "<p><<sacré bleu!>></p>" - expected = u"<<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>".encode("utf-8") - soup = self.soup(text) - str = soup.p.string - #self.assertEquals(str.encode("utf-8"), expected) - - def test_br_tag_is_empty_element(self): - """A <br> tag is designated as an empty-element tag.""" - soup = self.soup("<br></br>") - self.assertTrue(soup.br.is_empty_element) - self.assertEquals(str(soup.br), "<br />") - - def test_p_tag_is_not_empty_element(self): - """A <p> tag is not designated as an empty-element tag.""" - soup = self.soup("<p />") - self.assertFalse(soup.p.is_empty_element) - self.assertEquals(str(soup.p), "<p></p>") - - def test_soupstrainer(self): - strainer = SoupStrainer("b") - soup = self.soup("A <b>bold</b> <meta /> <i>statement</i>", - parse_only=strainer) - self.assertEquals(soup.decode(), "<b>bold</b>") - - -class TestLXMLBuilderInvalidMarkup(SoupTest): - """Tests of invalid markup for the LXML tree builder. - - Subclass this to test other builders. - - These are very likely to give different results for different tree - builders. It's not required that a tree builder handle invalid - markup at all. - """ - - def test_table_containing_bare_markup(self): - # Markup should be in table cells, not directly in the table. - self.assertSoupEquals("<table><div>Foo</div></table>") - - def test_incorrectly_nested_table(self): - # The second <table> tag is floating in the <tr> tag - # rather than being inside a <td>. - bad_markup = ('<table id="1">' - '<tr>' - "<td>Here's another table:</td>" - '<table id="2">' - '<tr><td>foo</td></tr>' - '</table></td>') - - def test_unclosed_block_level_elements(self): - # Unclosed block-level elements should be closed. - self.assertSoupEquals( - '<blockquote><p><b>Foo</blockquote><p>Bar', - '<blockquote><p><b>Foo</b></p></blockquote><p>Bar</p>') - - def test_fake_self_closing_tag(self): - # If a self-closing tag presents as a normal tag, the 'open' - # tag is treated as an instance of the self-closing tag and - # the 'close' tag is ignored. - self.assertSoupEquals( - "<item><link>http://foo.com/</link></item>", - "<item><link />http://foo.com/</item>") - - def test_boolean_attribute_with_no_value_gets_empty_value(self): - soup = self.soup("<table><td nowrap>foo</td></table>") - self.assertEquals(soup.table.td['nowrap'], '') - - def test_incorrectly_nested_tables(self): - self.assertSoupEquals( - '<table><tr><table><tr id="nested">', - '<table><tr><table><tr id="nested"></tr></table></tr></table>') - - def test_paragraphs_containing_block_display_elements(self): - markup = self.soup("<p>this is the definition:" - "<dl><dt>first case</dt>") - # The <p> tag is closed before the <dl> tag begins. - self.assertEquals(markup.p.contents, ["this is the definition:"]) - - def test_empty_element_tag_with_contents(self): - self.assertSoupEquals("<br>foo</br>", "<br />foo") - - def test_doctype_in_body(self): - markup = "<p>one<!DOCTYPE foobar>two</p>" - self.assertSoupEquals(markup) - - def test_nonsensical_declaration(self): - # Declarations that don't make any sense are ignored. - self.assertSoupEquals('<! Foo = -8><p>a</p>', "<p>a</p>") - - def test_whitespace_in_doctype(self): - # A declaration that has extra whitespace is ignored. - self.assertSoupEquals( - ('<! DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">' - '<p>foo</p>'), - '<p>foo</p>') - - def test_incomplete_declaration(self): - # An incomplete declaration will screw up the rest of the document. - self.assertSoupEquals('a<!b <p>c', '<p>a</p>') - - def test_cdata_where_it_doesnt_belong(self): - #CDATA sections are ignored. - markup = "<div><![CDATA[foo]]>" - self.assertSoupEquals(markup, "<div></div>") - - def test_attribute_value_never_got_closed(self): - markup = '<a href="http://foo.com/</a> and blah and blah' - soup = self.soup(markup) - self.assertEquals( - soup.a['href'], "http://foo.com/</a> and blah and blah") - - def test_attribute_value_was_closed_by_subsequent_tag(self): - markup = """<a href="foo</a>, </a><a href="bar">baz</a>""" - soup = self.soup(markup) - # The string between the first and second quotes was interpreted - # as the value of the 'href' attribute. - self.assertEquals(soup.a['href'], 'foo</a>, </a><a href=') - - #The string after the second quote (bar"), was treated as an - #empty attribute called bar. - self.assertEquals(soup.a['bar'], '') - self.assertEquals(soup.a.string, "baz") - - def test_unquoted_attribute_value(self): - soup = self.soup('<a style={height:21px;}></a>') - self.assertEquals(soup.a['style'], '{height:21px;}') - - def test_attribute_value_with_embedded_brackets(self): - soup = self.soup('<a b="<a>">') - self.assertEquals(soup.a['b'], '<a>') - - def test_nonexistent_entity(self): - soup = self.soup("<p>foo&#bar;baz</p>") - self.assertEquals(soup.p.string, "foobar;baz") - - # Compare a real entity. - soup = self.soup("<p>foodbaz</p>") - self.assertEquals(soup.p.string, "foodbaz") - - # Also compare html5lib, which preserves the &# before the - # entity name. - - def test_entity_out_of_range(self): - # An entity that's out of range will be ignored. - soup = self.soup("<p>�</p>") - self.assertEquals(soup.p.string, None) - - soup = self.soup("<p>�</p>") - self.assertEquals(soup.p.string, None) - - - def test_entity_was_not_finished(self): - soup = self.soup("<p><Hello>") - # Compare html5lib, which completes the entity. - self.assertEquals(soup.p.string, "<Hello>") - - def test_document_ends_with_incomplete_declaration(self): - soup = self.soup('<p>a<!b') - # This becomes a string 'a'. The incomplete declaration is ignored. - # Compare html5lib, which turns it into a comment. - self.assertEquals(soup.p.contents, ['a']) - - def test_document_starts_with_bogus_declaration(self): - soup = self.soup('<! Foo ><p>a</p>') - # The declaration is ignored altogether. - self.assertEquals(soup.encode(), "<html><body><p>a</p></body></html>") - - def test_tag_name_contains_unicode(self): - # Unicode characters in tag names are stripped. - tag_name = u"<our\N{SNOWMAN}>Joe</our\N{SNOWMAN}>" - self.assertSoupEquals("<our>Joe</our>") - -class TestLXMLBuilderEncodingConversion(SoupTest): - # Test Beautiful Soup's ability to decode and encode from various - # encodings. - - def setUp(self): - super(TestLXMLBuilderEncodingConversion, self).setUp() - self.unicode_data = u"<html><head></head><body><foo>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</foo></body></html>" - self.utf8_data = self.unicode_data.encode("utf-8") - # Just so you know what it looks like. - self.assertEqual( - self.utf8_data, - "<html><head></head><body><foo>Sacr\xc3\xa9 bleu!</foo></body></html>") - - def test_ascii_in_unicode_out(self): - # ASCII input is converted to Unicode. The original_encoding - # attribute is set. - ascii = "<foo>a</foo>" - soup_from_ascii = self.soup(ascii) - unicode_output = soup_from_ascii.decode() - self.assertTrue(isinstance(unicode_output, unicode)) - self.assertEquals(unicode_output, self.document_for(ascii)) - self.assertEquals(soup_from_ascii.original_encoding, "ascii") - - def test_unicode_in_unicode_out(self): - # Unicode input is left alone. The original_encoding attribute - # is not set. - soup_from_unicode = self.soup(self.unicode_data) - self.assertEquals(soup_from_unicode.decode(), self.unicode_data) - self.assertEquals(soup_from_unicode.foo.string, u'Sacr\xe9 bleu!') - self.assertEquals(soup_from_unicode.original_encoding, None) - - def test_utf8_in_unicode_out(self): - # UTF-8 input is converted to Unicode. The original_encoding - # attribute is set. - soup_from_utf8 = self.soup(self.utf8_data) - self.assertEquals(soup_from_utf8.decode(), self.unicode_data) - self.assertEquals(soup_from_utf8.foo.string, u'Sacr\xe9 bleu!') - - def test_utf8_out(self): - # The internal data structures can be encoded as UTF-8. - soup_from_unicode = self.soup(self.unicode_data) - self.assertEquals(soup_from_unicode.encode('utf-8'), self.utf8_data) - - HEBREW_DOCUMENT = '<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>' - - def test_real_hebrew_document(self): - # A real-world test to make sure we can convert ISO-8859-9 (a - # Hebrew encoding) to UTF-8. - soup = self.soup(self.HEBREW_DOCUMENT, - from_encoding="iso-8859-8") - self.assertEquals(soup.original_encoding, 'iso-8859-8') - self.assertEquals( - soup.encode('utf-8'), - self.HEBREW_DOCUMENT.decode("iso-8859-8").encode("utf-8")) - - -class TestLXMLXMLBuilder(SoupTest): - """Test XML-specific parsing behavior. - - Most of the tests use HTML as an example, since Beautiful Soup is - mainly an HTML parser. This test suite is a base for XML-specific - tree builders. - """ - - @property - def default_builder(self): - return LXMLTreeBuilderForXML() - - def test_mixed_case_tags(self): - # Mixed-case tags are *not* folded to lowercase, but the - # end tag is always the same case as the start tag. - self.assertSoupEquals( - "<a><B><Cd><EFG /></CD></b></A>", - "<a><B><Cd><EFG /></Cd></B></a>") - - - def test_cdata_becomes_text(self): - # LXML sends CData sections as 'data' events, so we can't - # create special CData objects for them. We have to use - # NavigableString. I would like to fix this, but it's not a - # very high priority. - markup = "<foo><![CDATA[iamcdata]]></foo>" - soup = self.soup(markup) - cdata = soup.foo.contents[0] - self.assertEquals(cdata.__class__.__name__, 'NavigableString') - - - def test_can_handle_invalid_xml(self): - self.assertSoupEquals("<a><b>", "<a><b /></a>") - - def test_empty_element_tag(self): - soup = self.soup("<p><iamselfclosing /></p>") - self.assertTrue(soup.iamselfclosing.is_empty_element) - - def test_self_empty_tag_treated_as_empty_element(self): - soup = self.soup("<p><iamclosed></iamclosed></p>") - self.assertTrue(soup.iamclosed.is_empty_element) - - def test_self_nonempty_tag_is_not_empty_element(self): - soup = self.soup("<p><ihavecontents>contents</ihavecontents></p>") - self.assertFalse(soup.ihavecontents.is_empty_element) - - def test_empty_tag_that_stops_being_empty_gets_a_closing_tag(self): - soup = self.soup("<bar />") - self.assertTrue(soup.bar.is_empty_element) - soup.bar.insert(1, "Contents") - self.assertFalse(soup.bar.is_empty_element) - self.assertEquals(str(soup), self.document_for("<bar>Contents</bar>")) - - def test_designated_empty_element_tag_has_no_closing_tag(self): - builder = LXMLTreeBuilderForXML(empty_element_tags=['bar']) - soup = BeautifulSoup(builder=builder, markup="<bar></bar>") - self.assertTrue(soup.bar.is_empty_element) - self.assertEquals(str(soup), self.document_for("<bar />")) - - def test_empty_tag_not_in_empty_element_tag_list_has_closing_tag(self): - builder = LXMLTreeBuilderForXML(empty_element_tags=['bar']) - - soup = BeautifulSoup(builder=builder, markup="<foo />") - self.assertFalse(soup.foo.is_empty_element) - self.assertEquals(str(soup), self.document_for("<foo></foo>")) - - def test_designated_empty_element_tag_does_not_change_parser_behavior(self): - # The designated list of empty-element tags only affects how - # empty tags are presented. It does not affect how tags are - # parsed--that's the parser's job. - builder = LXMLTreeBuilderForXML(empty_element_tags=['bar']) - soup = BeautifulSoup(builder=builder, markup="<bar>contents</bar>") - self.assertEquals(str(soup), self.document_for("<bar>contents</bar>")) diff --git a/tests/test_soup.py b/tests/test_soup.py deleted file mode 100644 index 87d6f3b..0000000 --- a/tests/test_soup.py +++ /dev/null @@ -1,130 +0,0 @@ -# -*- coding: utf-8 -*- -"""Tests of Beautiful Soup as a whole.""" - -import unittest -from bs4.element import SoupStrainer -from bs4.dammit import EntitySubstitution, UnicodeDammit -from bs4.testing import SoupTest - - -class TestSelectiveParsing(SoupTest): - - def test_parse_with_soupstrainer(self): - markup = "No<b>Yes</b><a>No<b>Yes <c>Yes</c></b>" - strainer = SoupStrainer("b") - soup = self.soup(markup, parse_only=strainer) - self.assertEquals(soup.encode(), "<b>Yes</b><b>Yes <c>Yes</c></b>") - - -class TestEntitySubstitution(unittest.TestCase): - """Standalone tests of the EntitySubstitution class.""" - def setUp(self): - self.sub = EntitySubstitution - - def test_simple_html_substitution(self): - # Unicode characters corresponding to named HTML entites - # are substituted, and no others. - s = u"foo\u2200\N{SNOWMAN}\u00f5bar" - self.assertEquals(self.sub.substitute_html(s), - u"foo∀\N{SNOWMAN}õbar") - - def test_smart_quote_substitution(self): - # MS smart quotes are a common source of frustration, so we - # give them a special test. - quotes = "\x91\x92foo\x93\x94" - dammit = UnicodeDammit(quotes) - self.assertEquals(self.sub.substitute_html(dammit.markup), - "‘’foo“”") - - def test_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_false(self): - s = 'Welcome to "my bar"' - self.assertEquals(self.sub.substitute_xml(s, False), s) - - def test_xml_attribute_quoting_normally_uses_double_quotes(self): - self.assertEquals(self.sub.substitute_xml("Welcome", True), - '"Welcome"') - self.assertEquals(self.sub.substitute_xml("Bob's Bar", True), - '"Bob\'s Bar"') - - def test_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotes(self): - s = 'Welcome to "my bar"' - self.assertEquals(self.sub.substitute_xml(s, True), - "'Welcome to \"my bar\"'") - - def test_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotes(self): - s = 'Welcome to "Bob\'s Bar"' - self.assertEquals( - self.sub.substitute_xml(s, True), - '"Welcome to "Bob\'s Bar""') - - def test_xml_quotes_arent_escaped_when_value_is_not_being_quoted(self): - quoted = 'Welcome to "Bob\'s Bar"' - self.assertEquals(self.sub.substitute_xml(quoted), quoted) - - def test_xml_quoting_handles_angle_brackets(self): - self.assertEquals( - self.sub.substitute_xml("foo<bar>"), - "foo<bar>") - - def test_xml_quoting_handles_ampersands(self): - self.assertEquals(self.sub.substitute_xml("AT&T"), "AT&T") - - def test_xml_quoting_ignores_ampersands_when_they_are_part_of_an_entity(self): - self.assertEquals( - self.sub.substitute_xml("ÁT&T"), - "ÁT&T") - - def test_quotes_not_html_substituted(self): - """There's no need to do this except inside attribute values.""" - text = 'Bob\'s "bar"' - self.assertEquals(self.sub.substitute_html(text), text) - -class TestUnicodeDammit(unittest.TestCase): - """Standalone tests of Unicode, Dammit.""" - - def test_smart_quotes_to_unicode(self): - markup = "<foo>\x91\x92\x93\x94</foo>" - dammit = UnicodeDammit(markup) - self.assertEquals( - dammit.unicode_markup, u"<foo>\u2018\u2019\u201c\u201d</foo>") - - def test_smart_quotes_to_xml_entities(self): - markup = "<foo>\x91\x92\x93\x94</foo>" - dammit = UnicodeDammit(markup, smart_quotes_to="xml") - self.assertEquals( - dammit.unicode_markup, "<foo>‘’“”</foo>") - - def test_smart_quotes_to_html_entities(self): - markup = "<foo>\x91\x92\x93\x94</foo>" - dammit = UnicodeDammit(markup, smart_quotes_to="html") - self.assertEquals( - dammit.unicode_markup, "<foo>‘’“”</foo>") - - def test_detect_utf8(self): - utf8 = "\xc3\xa9" - dammit = UnicodeDammit(utf8) - self.assertEquals(dammit.unicode_markup, u'\xe9') - self.assertEquals(dammit.original_encoding, 'utf-8') - - def test_convert_hebrew(self): - hebrew = "\xed\xe5\xec\xf9" - dammit = UnicodeDammit(hebrew, ["iso-8859-8"]) - self.assertEquals(dammit.original_encoding, 'iso-8859-8') - self.assertEquals(dammit.unicode_markup, u'\u05dd\u05d5\u05dc\u05e9') - - def test_dont_see_smart_quotes_where_there_are_none(self): - utf_8 = "\343\202\261\343\203\274\343\202\277\343\202\244 Watch" - dammit = UnicodeDammit(utf_8) - self.assertEquals(dammit.original_encoding, 'utf-8') - self.assertEquals(dammit.unicode_markup.encode("utf-8"), utf_8) - - def test_ignore_inappropriate_codecs(self): - utf8_data = u"Räksmörgås".encode("utf-8") - dammit = UnicodeDammit(utf8_data, ["iso-8859-8"]) - self.assertEquals(dammit.original_encoding, 'utf-8') - - def test_ignore_invalid_codecs(self): - utf8_data = u"Räksmörgås".encode("utf-8") - for bad_encoding in ['.utf8', '...', 'utF---16.!']: - dammit = UnicodeDammit(utf8_data, [bad_encoding]) - self.assertEquals(dammit.original_encoding, 'utf-8') diff --git a/tests/test_tree.py b/tests/test_tree.py deleted file mode 100644 index 8d0027c..0000000 --- a/tests/test_tree.py +++ /dev/null @@ -1,982 +0,0 @@ -# -*- coding: utf-8 -*- -"""Tests for Beautiful Soup's tree traversal methods. - -The tree traversal methods are the main advantage of using Beautiful -Soup over other parsers. - -Different parsers will build different Beautiful Soup trees given the -same markup, but all Beautiful Soup trees can be traversed with the -methods tested here. -""" - -import copy -import pickle -import re -from bs4 import BeautifulSoup -from bs4.builder import builder_registry -from bs4.element import CData, SoupStrainer, Tag -from bs4.testing import SoupTest - -class TreeTest(SoupTest): - - def assertSelects(self, tags, should_match): - """Make sure that the given tags have the correct text. - - This is used in tests that define a bunch of tags, each - containing a single string, and then select certain strings by - some mechanism. - """ - self.assertEqual([tag.string for tag in tags], should_match) - - def assertSelectsIDs(self, tags, should_match): - """Make sure that the given tags have the correct IDs. - - This is used in tests that define a bunch of tags, each - containing a single string, and then select certain strings by - some mechanism. - """ - self.assertEqual([tag['id'] for tag in tags], should_match) - - -class TestFind(TreeTest): - """Basic tests of the find() method. - - find() just calls find_all() with limit=1, so it's not tested all - that thouroughly here. - """ - - def test_find_tag(self): - soup = self.soup("<a>1</a><b>2</b><a>3</a><b>4</b>") - self.assertEqual(soup.find("b").string, "2") - - def test_unicode_text_find(self): - soup = self.soup(u'<h1>Räksmörgås</h1>') - self.assertEqual(soup.find(text=u'Räksmörgås'), u'Räksmörgås') - - -class TestFindAll(TreeTest): - """Basic tests of the find_all() method.""" - - def test_find_all_text_nodes(self): - """You can search the tree for text nodes.""" - soup = self.soup("<html>Foo<b>bar</b>\xbb</html>") - # Exact match. - self.assertEqual(soup.find_all(text="bar"), [u"bar"]) - # Match any of a number of strings. - self.assertEqual( - soup.find_all(text=["Foo", "bar"]), [u"Foo", u"bar"]) - # Match a regular expression. - self.assertEqual(soup.find_all(text=re.compile('.*')), - [u"Foo", u"bar", u'\xbb']) - # Match anything. - self.assertEqual(soup.find_all(text=True), - [u"Foo", u"bar", u'\xbb']) - - def test_find_all_limit(self): - """You can limit the number of items returned by find_all.""" - soup = self.soup("<a>1</a><a>2</a><a>3</a><a>4</a><a>5</a>") - self.assertSelects(soup.find_all('a', limit=3), ["1", "2", "3"]) - self.assertSelects(soup.find_all('a', limit=1), ["1"]) - self.assertSelects( - soup.find_all('a', limit=10), ["1", "2", "3", "4", "5"]) - - # A limit of 0 means no limit. - self.assertSelects( - soup.find_all('a', limit=0), ["1", "2", "3", "4", "5"]) - -class TestFindAllByName(TreeTest): - """Test ways of finding tags by tag name.""" - - def setUp(self): - super(TreeTest, self).setUp() - self.tree = self.soup("""<a>First tag.</a> - <b>Second tag.</b> - <c>Third <a>Nested tag.</a> tag.</c>""") - - def test_find_all_by_tag_name(self): - # Find all the <a> tags. - self.assertSelects( - self.tree.find_all('a'), ['First tag.', 'Nested tag.']) - - def test_find_all_on_non_root_element(self): - # You can call find_all on any node, not just the root. - self.assertSelects(self.tree.c.find_all('a'), ['Nested tag.']) - - def test_calling_element_invokes_find_all(self): - self.assertSelects(self.tree('a'), ['First tag.', 'Nested tag.']) - - def test_find_all_by_tag_strainer(self): - self.assertSelects( - self.tree.find_all(SoupStrainer('a')), - ['First tag.', 'Nested tag.']) - - def test_find_all_by_tag_names(self): - self.assertSelects( - self.tree.find_all(['a', 'b']), - ['First tag.', 'Second tag.', 'Nested tag.']) - - def test_find_all_by_tag_dict(self): - self.assertSelects( - self.tree.find_all({'a' : True, 'b' : True}), - ['First tag.', 'Second tag.', 'Nested tag.']) - - def test_find_all_by_tag_re(self): - self.assertSelects( - self.tree.find_all(re.compile('^[ab]$')), - ['First tag.', 'Second tag.', 'Nested tag.']) - - def test_find_all_with_tags_matching_method(self): - # You can define an oracle method that determines whether - # a tag matches the search. - def id_matches_name(tag): - return tag.name == tag.get('id') - - tree = self.soup("""<a id="a">Match 1.</a> - <a id="1">Does not match.</a> - <b id="b">Match 2.</a>""") - - self.assertSelects( - tree.find_all(id_matches_name), ["Match 1.", "Match 2."]) - - -class TestFindAllByAttribute(TreeTest): - - def test_find_all_by_attribute_name(self): - # You can pass in keyword arguments to find_all to search by - # attribute. - tree = self.soup(""" - <a id="first">Matching a.</a> - <a id="second"> - Non-matching <b id="first">Matching b.</b>a. - </a>""") - self.assertSelects(tree.find_all(id='first'), - ["Matching a.", "Matching b."]) - - def test_find_all_by_attribute_dict(self): - # You can pass in a dictionary as the argument 'attrs'. This - # lets you search for attributes like 'name' (a fixed argument - # to find_all) and 'class' (a reserved word in Python.) - tree = self.soup(""" - <a name="name1" class="class1">Name match.</a> - <a name="name2" class="class2">Class match.</a> - <a name="name3" class="class3">Non-match.</a> - <name1>A tag called 'name1'.</name1> - """) - - # This doesn't do what you want. - self.assertSelects(tree.find_all(name='name1'), - ["A tag called 'name1'."]) - # This does what you want. - self.assertSelects(tree.find_all(attrs={'name' : 'name1'}), - ["Name match."]) - - # Passing class='class2' would cause a syntax error. - self.assertSelects(tree.find_all(attrs={'class' : 'class2'}), - ["Class match."]) - - def test_find_all_by_class(self): - # Passing in a string to 'attrs' will search the CSS class. - tree = self.soup(""" - <a class="1">Class 1.</a> - <a class="2">Class 2.</a> - <b class="1">Class 1.</b> - <c class="3 4">Class 3 and 4.</c> - """) - self.assertSelects(tree.find_all('a', '1'), ['Class 1.']) - self.assertSelects(tree.find_all(attrs='1'), ['Class 1.', 'Class 1.']) - self.assertSelects(tree.find_all('c', '3'), ['Class 3 and 4.']) - self.assertSelects(tree.find_all('c', '4'), ['Class 3 and 4.']) - - def test_find_all_by_attribute_soupstrainer(self): - tree = self.soup(""" - <a id="first">Match.</a> - <a id="second">Non-match.</a>""") - - strainer = SoupStrainer(attrs={'id' : 'first'}) - self.assertSelects(tree.find_all(strainer), ['Match.']) - - def test_find_all_with_missing_atribute(self): - # You can pass in None as the value of an attribute to find_all. - # This will match tags that do not have that attribute set. - tree = self.soup("""<a id="1">ID present.</a> - <a>No ID present.</a> - <a id="">ID is empty.</a>""") - self.assertSelects(tree.find_all('a', id=None), ["No ID present."]) - - def test_find_all_with_defined_attribute(self): - # You can pass in None as the value of an attribute to find_all. - # This will match tags that have that attribute set to any value. - tree = self.soup("""<a id="1">ID present.</a> - <a>No ID present.</a> - <a id="">ID is empty.</a>""") - self.assertSelects( - tree.find_all(id=True), ["ID present.", "ID is empty."]) - - def test_find_all_with_numeric_attribute(self): - # If you search for a number, it's treated as a string. - tree = self.soup("""<a id=1>Unquoted attribute.</a> - <a id="1">Quoted attribute.</a>""") - - expected = ["Unquoted attribute.", "Quoted attribute."] - self.assertSelects(tree.find_all(id=1), expected) - self.assertSelects(tree.find_all(id="1"), expected) - - def test_find_all_with_list_attribute_values(self): - # You can pass a list of attribute values instead of just one, - # and you'll get tags that match any of the values. - tree = self.soup("""<a id="1">1</a> - <a id="2">2</a> - <a id="3">3</a> - <a>No ID.</a>""") - self.assertSelects(tree.find_all(id=["1", "3", "4"]), - ["1", "3"]) - - def test_find_all_with_regular_expression_attribute_value(self): - # You can pass a regular expression as an attribute value, and - # you'll get tags whose values for that attribute match the - # regular expression. - tree = self.soup("""<a id="a">One a.</a> - <a id="aa">Two as.</a> - <a id="ab">Mixed as and bs.</a> - <a id="b">One b.</a> - <a>No ID.</a>""") - - self.assertSelects(tree.find_all(id=re.compile("^a+$")), - ["One a.", "Two as."]) - - -class TestIndex(TreeTest): - """Test Tag.index""" - def test_index(self): - tree = self.soup("""<wrap> - <a>Identical</a> - <b>Not identical</b> - <a>Identical</a> - - <c><d>Identical with child</d></c> - <b>Also not identical</b> - <c><d>Identical with child</d></c> - </wrap>""") - wrap = tree.wrap - for i, element in enumerate(wrap.contents): - self.assertEqual(i, wrap.index(element)) - self.assertRaises(ValueError, tree.index, 1) - - -class TestParentOperations(TreeTest): - """Test navigation and searching through an element's parents.""" - - def setUp(self): - super(TestParentOperations, self).setUp() - self.tree = self.soup('''<ul id="empty"></ul> - <ul id="top"> - <ul id="middle"> - <ul id="bottom"> - <b>Start here</b> - </ul> - </ul>''') - self.start = self.tree.b - - - def test_parent(self): - self.assertEquals(self.start.parent['id'], 'bottom') - self.assertEquals(self.start.parent.parent['id'], 'middle') - self.assertEquals(self.start.parent.parent.parent['id'], 'top') - - def test_parent_of_top_tag_is_soup_object(self): - top_tag = self.tree.contents[0] - self.assertEquals(top_tag.parent, self.tree) - - def test_soup_object_has_no_parent(self): - self.assertEquals(None, self.tree.parent) - - def test_find_parents(self): - self.assertSelectsIDs( - self.start.find_parents('ul'), ['bottom', 'middle', 'top']) - self.assertSelectsIDs( - self.start.find_parents('ul', id="middle"), ['middle']) - - def test_find_parent(self): - self.assertEquals(self.start.find_parent('ul')['id'], 'bottom') - - def test_parent_of_text_element(self): - text = self.tree.find(text="Start here") - self.assertEquals(text.parent.name, 'b') - - def test_text_element_find_parent(self): - text = self.tree.find(text="Start here") - self.assertEquals(text.find_parent('ul')['id'], 'bottom') - - def test_parent_generator(self): - parents = [parent['id'] for parent in self.start.parents - if parent is not None and 'id' in parent.attrs] - self.assertEquals(parents, ['bottom', 'middle', 'top']) - - -class ProximityTest(TreeTest): - - def setUp(self): - super(TreeTest, self).setUp() - self.tree = self.soup( - '<html id="start"><head></head><body><b id="1">One</b><b id="2">Two</b><b id="3">Three</b></body></html>') - - -class TestNextOperations(ProximityTest): - - def setUp(self): - super(TestNextOperations, self).setUp() - self.start = self.tree.b - - def test_next(self): - self.assertEquals(self.start.next_element, "One") - self.assertEquals(self.start.next_element.next_element['id'], "2") - - def test_next_of_last_item_is_none(self): - last = self.tree.find(text="Three") - self.assertEquals(last.next_element, None) - - def test_next_of_root_is_none(self): - # The document root is outside the next/previous chain. - self.assertEquals(self.tree.next_element, None) - - def test_find_all_next(self): - self.assertSelects(self.start.find_all_next('b'), ["Two", "Three"]) - self.start.find_all_next(id=3) - self.assertSelects(self.start.find_all_next(id=3), ["Three"]) - - def test_find_next(self): - self.assertEquals(self.start.find_next('b')['id'], '2') - self.assertEquals(self.start.find_next(text="Three"), "Three") - - def test_find_next_for_text_element(self): - text = self.tree.find(text="One") - self.assertEquals(text.find_next("b").string, "Two") - self.assertSelects(text.find_all_next("b"), ["Two", "Three"]) - - def test_next_generator(self): - start = self.tree.find(text="Two") - successors = [node for node in start.next_elements] - # There are two successors: the final <b> tag and its text contents. - # Then we go off the end. - tag, contents, none = successors - self.assertEquals(tag['id'], '3') - self.assertEquals(contents, "Three") - self.assertEquals(none, None) - - # XXX Should next_elements really return None? Seems like it - # should just stop. - - -class TestPreviousOperations(ProximityTest): - - def setUp(self): - super(TestPreviousOperations, self).setUp() - self.end = self.tree.find(text="Three") - - def test_previous(self): - self.assertEquals(self.end.previous_element['id'], "3") - self.assertEquals(self.end.previous_element.previous_element, "Two") - - def test_previous_of_first_item_is_none(self): - first = self.tree.find('html') - self.assertEquals(first.previous_element, None) - - def test_previous_of_root_is_none(self): - # The document root is outside the next/previous chain. - # XXX This is broken! - #self.assertEquals(self.tree.previous_element, None) - pass - - def test_find_all_previous(self): - # The <b> tag containing the "Three" node is the predecessor - # of the "Three" node itself, which is why "Three" shows up - # here. - self.assertSelects( - self.end.find_all_previous('b'), ["Three", "Two", "One"]) - self.assertSelects(self.end.find_all_previous(id=1), ["One"]) - - def test_find_previous(self): - self.assertEquals(self.end.find_previous('b')['id'], '3') - self.assertEquals(self.end.find_previous(text="One"), "One") - - def test_find_previous_for_text_element(self): - text = self.tree.find(text="Three") - self.assertEquals(text.find_previous("b").string, "Three") - self.assertSelects( - text.find_all_previous("b"), ["Three", "Two", "One"]) - - def test_previous_generator(self): - start = self.tree.find(text="One") - predecessors = [node for node in start.previous_elements] - - # There are four predecessors: the <b> tag containing "One" - # the <body> tag, the <head> tag, and the <html> tag. Then we - # go off the end. - b, body, head, html, none = predecessors - self.assertEquals(b['id'], '1') - self.assertEquals(body.name, "body") - self.assertEquals(head.name, "head") - self.assertEquals(html.name, "html") - self.assertEquals(none, None) - - # Again, we shouldn't be returning None. - - -class SiblingTest(TreeTest): - - def setUp(self): - super(SiblingTest, self).setUp() - markup = '''<html> - <span id="1"> - <span id="1.1"></span> - </span> - <span id="2"> - <span id="2.1"></span> - </span> - <span id="3"> - <span id="3.1"></span> - </span> - <span id="4"></span> - </html>''' - # All that whitespace looks good but makes the tests more - # difficult. Get rid of it. - markup = re.compile("\n\s*").sub("", markup) - self.tree = self.soup(markup) - - -class TestNextSibling(SiblingTest): - - def setUp(self): - super(TestNextSibling, self).setUp() - self.start = self.tree.find(id="1") - - def test_next_sibling_of_root_is_none(self): - self.assertEquals(self.tree.next_sibling, None) - - def test_next_sibling(self): - self.assertEquals(self.start.next_sibling['id'], '2') - self.assertEquals(self.start.next_sibling.next_sibling['id'], '3') - - # Note the difference between next_sibling and next_element. - self.assertEquals(self.start.next_element['id'], '1.1') - - def test_next_sibling_may_not_exist(self): - self.assertEquals(self.tree.html.next_sibling, None) - - nested_span = self.tree.find(id="1.1") - self.assertEquals(nested_span.next_sibling, None) - - last_span = self.tree.find(id="4") - self.assertEquals(last_span.next_sibling, None) - - def test_find_next_sibling(self): - self.assertEquals(self.start.find_next_sibling('span')['id'], '2') - - def test_next_siblings(self): - self.assertSelectsIDs(self.start.find_next_siblings("span"), - ['2', '3', '4']) - - self.assertSelectsIDs(self.start.find_next_siblings(id='3'), ['3']) - - def test_next_sibling_for_text_element(self): - soup = self.soup("Foo<b>bar</b>baz") - start = soup.find(text="Foo") - self.assertEquals(start.next_sibling.name, 'b') - self.assertEquals(start.next_sibling.next_sibling, 'baz') - - self.assertSelects(start.find_next_siblings('b'), ['bar']) - self.assertEquals(start.find_next_sibling(text="baz"), "baz") - self.assertEquals(start.find_next_sibling(text="nonesuch"), None) - - -class TestPreviousSibling(SiblingTest): - - def setUp(self): - super(TestPreviousSibling, self).setUp() - self.end = self.tree.find(id="4") - - def test_previous_sibling_of_root_is_none(self): - self.assertEquals(self.tree.previous_sibling, None) - - def test_previous_sibling(self): - self.assertEquals(self.end.previous_sibling['id'], '3') - self.assertEquals(self.end.previous_sibling.previous_sibling['id'], '2') - - # Note the difference between previous_sibling and previous_element. - self.assertEquals(self.end.previous_element['id'], '3.1') - - def test_previous_sibling_may_not_exist(self): - self.assertEquals(self.tree.html.previous_sibling, None) - - nested_span = self.tree.find(id="1.1") - self.assertEquals(nested_span.previous_sibling, None) - - first_span = self.tree.find(id="1") - self.assertEquals(first_span.previous_sibling, None) - - def test_find_previous_sibling(self): - self.assertEquals(self.end.find_previous_sibling('span')['id'], '3') - - def test_previous_siblings(self): - self.assertSelectsIDs(self.end.find_previous_siblings("span"), - ['3', '2', '1']) - - self.assertSelectsIDs(self.end.find_previous_siblings(id='1'), ['1']) - - def test_previous_sibling_for_text_element(self): - soup = self.soup("Foo<b>bar</b>baz") - start = soup.find(text="baz") - self.assertEquals(start.previous_sibling.name, 'b') - self.assertEquals(start.previous_sibling.previous_sibling, 'Foo') - - self.assertSelects(start.find_previous_siblings('b'), ['bar']) - self.assertEquals(start.find_previous_sibling(text="Foo"), "Foo") - self.assertEquals(start.find_previous_sibling(text="nonesuch"), None) - - -class TestTreeModification(SoupTest): - - def test_attribute_modification(self): - soup = self.soup('<a id="1"></a>') - soup.a['id'] = 2 - self.assertEqual(soup.decode(), self.document_for('<a id="2"></a>')) - del(soup.a['id']) - self.assertEqual(soup.decode(), self.document_for('<a></a>')) - soup.a['id2'] = 'foo' - self.assertEqual(soup.decode(), self.document_for('<a id2="foo"></a>')) - - def test_new_tag_creation(self): - builder = builder_registry.lookup('html5lib')() - soup = self.soup("<body></body>", builder=builder) - a = Tag(soup, builder, 'a') - ol = Tag(soup, builder, 'ol') - a['href'] = 'http://foo.com/' - soup.body.insert(0, a) - soup.body.insert(1, ol) - self.assertEqual( - soup.body.encode(), - '<body><a href="http://foo.com/"></a><ol></ol></body>') - - def test_append_to_contents_moves_tag(self): - doc = """<p id="1">Don't leave me <b>here</b>.</p> - <p id="2">Don\'t leave!</p>""" - soup = self.soup(doc) - second_para = soup.find(id='2') - bold = soup.b - - # Move the <b> tag to the end of the second paragraph. - soup.find(id='2').append(soup.b) - - # The <b> tag is now a child of the second paragraph. - self.assertEqual(bold.parent, second_para) - - self.assertEqual( - soup.decode(), self.document_for( - '<p id="1">Don\'t leave me .</p>\n' - '<p id="2">Don\'t leave!<b>here</b></p>')) - - def test_replace_tag_with_itself(self): - text = "<a><b></b><c>Foo<d></d></c></a><a><e></e></a>" - soup = self.soup(text) - c = soup.c - soup.c.replace_with(c) - self.assertEquals(soup.decode(), self.document_for(text)) - - def test_replace_final_node(self): - soup = self.soup("<b>Argh!</b>") - soup.find(text="Argh!").replace_with("Hooray!") - new_text = soup.find(text="Hooray!") - b = soup.b - self.assertEqual(new_text.previous_element, b) - self.assertEqual(new_text.parent, b) - self.assertEqual(new_text.previous_element.next_element, new_text) - self.assertEqual(new_text.next_element, None) - - def test_consecutive_text_nodes(self): - # A builder should never create two consecutive text nodes, - # but if you insert one next to another, Beautiful Soup will - # handle it correctly. - soup = self.soup("<a><b>Argh!</b><c></c></a>") - soup.b.insert(1, "Hooray!") - - self.assertEqual( - soup.decode(), self.document_for( - "<a><b>Argh!Hooray!</b><c></c></a>")) - - new_text = soup.find(text="Hooray!") - self.assertEqual(new_text.previous_element, "Argh!") - self.assertEqual(new_text.previous_element.next_element, new_text) - - self.assertEqual(new_text.previous_sibling, "Argh!") - self.assertEqual(new_text.previous_sibling.next_sibling, new_text) - - self.assertEqual(new_text.next_sibling, None) - self.assertEqual(new_text.next_element, soup.c) - - def test_insert_tag(self): - builder = self.default_builder - soup = self.soup( - "<a><b>Find</b><c>lady!</c><d></d></a>", builder=builder) - magic_tag = Tag(soup, builder, 'magictag') - magic_tag.insert(0, "the") - soup.a.insert(1, magic_tag) - - self.assertEqual( - soup.decode(), self.document_for( - "<a><b>Find</b><magictag>the</magictag><c>lady!</c><d></d></a>")) - - # Make sure all the relationships are hooked up correctly. - b_tag = soup.b - self.assertEqual(b_tag.next_sibling, magic_tag) - self.assertEqual(magic_tag.previous_sibling, b_tag) - - find = b_tag.find(text="Find") - self.assertEqual(find.next_element, magic_tag) - self.assertEqual(magic_tag.previous_element, find) - - c_tag = soup.c - self.assertEqual(magic_tag.next_sibling, c_tag) - self.assertEqual(c_tag.previous_sibling, magic_tag) - - the = magic_tag.find(text="the") - self.assertEqual(the.parent, magic_tag) - self.assertEqual(the.next_element, c_tag) - self.assertEqual(c_tag.previous_element, the) - - def test_insert_works_on_empty_element_tag(self): - # This is a little strange, since most HTML parsers don't allow - # markup like this to come through. But in general, we don't - # know what the parser would or wouldn't have allowed, so - # I'm letting this succeed for now. - soup = self.soup("<br />") - soup.br.insert(1, "Contents") - self.assertEquals(str(soup.br), "<br>Contents</br>") - - def test_replace_with(self): - soup = self.soup( - "<p>There's <b>no</b> business like <b>show</b> business</p>") - no, show = soup.find_all('b') - show.replace_with(no) - self.assertEquals( - soup.decode(), - self.document_for( - "<p>There's business like <b>no</b> business</p>")) - - self.assertEquals(show.parent, None) - self.assertEquals(no.parent, soup.p) - self.assertEquals(no.next_element, "no") - self.assertEquals(no.next_sibling, " business") - - def test_nested_tag_replace_with(self): - soup = self.soup( - """<a>We<b>reserve<c>the</c><d>right</d></b></a><e>to<f>refuse</f><g>service</g></e>""") - - # Replace the entire <b> tag and its contents ("reserve the - # right") with the <f> tag ("refuse"). - remove_tag = soup.b - move_tag = soup.f - remove_tag.replace_with(move_tag) - - self.assertEqual( - soup.decode(), self.document_for( - "<a>We<f>refuse</f></a><e>to<g>service</g></e>")) - - # The <b> tag is now an orphan. - self.assertEqual(remove_tag.parent, None) - self.assertEqual(remove_tag.find(text="right").next_element, None) - self.assertEqual(remove_tag.previous_element, None) - self.assertEqual(remove_tag.next_sibling, None) - self.assertEqual(remove_tag.previous_sibling, None) - - # The <f> tag is now connected to the <a> tag. - self.assertEqual(move_tag.parent, soup.a) - self.assertEqual(move_tag.previous_element, "We") - self.assertEqual(move_tag.next_element.next_element, soup.e) - self.assertEqual(move_tag.next_sibling, None) - - # The gap where the <f> tag used to be has been mended, and - # the word "to" is now connected to the <g> tag. - to_text = soup.find(text="to") - g_tag = soup.g - self.assertEqual(to_text.next_element, g_tag) - self.assertEqual(to_text.next_sibling, g_tag) - self.assertEqual(g_tag.previous_element, to_text) - self.assertEqual(g_tag.previous_sibling, to_text) - - def test_replace_with_children(self): - tree = self.soup(""" - <p>Unneeded <em>formatting</em> is unneeded</p> - """) - tree.em.replace_with_children() - self.assertEqual(tree.em, None) - self.assertEqual(tree.p.text, "Unneeded formatting is unneeded") - - def test_extract(self): - soup = self.soup( - '<html><body>Some content. <div id="nav">Nav crap</div> More content.</body></html>') - - self.assertEqual(len(soup.body.contents), 3) - extracted = soup.find(id="nav").extract() - - self.assertEqual( - soup.decode(), "<html><body>Some content. More content.</body></html>") - self.assertEqual(extracted.decode(), '<div id="nav">Nav crap</div>') - - # The extracted tag is now an orphan. - self.assertEqual(len(soup.body.contents), 2) - self.assertEqual(extracted.parent, None) - self.assertEqual(extracted.previous_element, None) - self.assertEqual(extracted.next_element.next_element, None) - - # The gap where the extracted tag used to be has been mended. - content_1 = soup.find(text="Some content. ") - content_2 = soup.find(text=" More content.") - self.assertEquals(content_1.next_element, content_2) - self.assertEquals(content_1.next_sibling, content_2) - self.assertEquals(content_2.previous_element, content_1) - self.assertEquals(content_2.previous_sibling, content_1) - - def test_clear(self): - """Tag.clear()""" - soup = self.soup("<p><a>String <em>Italicized</em></a> and another</p>") - # clear using extract() - a = soup.a - soup.p.clear() - self.assertEqual(len(soup.p.contents), 0) - self.assertTrue(hasattr(a, "contents")) - - # clear using decompose() - em = a.em - a.clear(decompose=True) - self.assertFalse(hasattr(em, "contents")) - - def test_string_set(self): - """Tag.string = 'string'""" - soup = self.soup("<a></a> <b><c></c></b>") - soup.a.string = "foo" - self.assertEqual(soup.a.contents, ["foo"]) - soup.b.string = "bar" - self.assertEqual(soup.b.contents, ["bar"]) - - -class TestElementObjects(SoupTest): - """Test various features of element objects.""" - - def test_len(self): - """The length of an element is its number of children.""" - soup = self.soup("<top>1<b>2</b>3</top>") - - # The BeautifulSoup object itself contains one element: the - # <top> tag. - self.assertEquals(len(soup.contents), 1) - self.assertEquals(len(soup), 1) - - # The <top> tag contains three elements: the text node "1", the - # <b> tag, and the text node "3". - self.assertEquals(len(soup.top), 3) - self.assertEquals(len(soup.top.contents), 3) - - def test_member_access_invokes_find(self): - """Accessing a Python member .foo or .fooTag invokes find('foo')""" - soup = self.soup('<b><i></i></b>') - self.assertEqual(soup.b, soup.find('b')) - self.assertEqual(soup.bTag, soup.find('b')) - self.assertEqual(soup.b.i, soup.find('b').find('i')) - self.assertEqual(soup.bTag.iTag, soup.find('b').find('i')) - self.assertEqual(soup.a, None) - self.assertEqual(soup.aTag, None) - - def test_has_attr(self): - """has_attr() checks for the presence of an attribute. - - Please note note: has_attr() is different from - __in__. has_attr() checks the tag's attributes and __in__ - checks the tag's chidlren. - """ - soup = self.soup("<foo attr='bar'>") - self.assertTrue(soup.foo.has_attr('attr')) - self.assertFalse(soup.foo.has_attr('attr2')) - - - def test_attributes_come_out_in_alphabetical_order(self): - markup = '<b a="1" z="5" m="3" f="2" y="4"></b>' - self.assertSoupEquals(markup, '<b a="1" f="2" m="3" y="4" z="5"></b>') - - def test_multiple_values_for_the_same_attribute_are_collapsed(self): - markup = '<b b="20" a="1" b="10" a="2" a="3" a="4"></b>' - self.assertSoupEquals(markup, '<b a="1" b="20"></b>') - - def test_string(self): - # A tag that contains only a text node makes that node - # available as .string. - soup = self.soup("<b>foo</b>") - self.assertEquals(soup.b.string, 'foo') - - def test_empty_tag_has_no_string(self): - # A tag with no children has no .stirng. - soup = self.soup("<b></b>") - self.assertEqual(soup.b.string, None) - - def test_tag_with_multiple_children_has_no_string(self): - # A tag with no children has no .string. - soup = self.soup("<a>foo<b></b><b></b></b>") - self.assertEqual(soup.b.string, None) - - soup = self.soup("<a>foo<b></b>bar</b>") - self.assertEqual(soup.b.string, None) - - # Even if all the children are strings, due to trickery, - # it won't work--but this would be a good optimization. - soup = self.soup("<a>foo</b>") - soup.a.insert(1, "bar") - self.assertEqual(soup.a.string, None) - - def test_tag_with_recursive_string_has_string(self): - # A tag with a single child which has a .string inherits that - # .string. - soup = self.soup("<a><b>foo</b></a>") - self.assertEqual(soup.a.string, "foo") - self.assertEqual(soup.string, "foo") - - def test_lack_of_string(self): - """Only a tag containing a single text node has a .string.""" - soup = self.soup("<b>f<i>e</i>o</b>") - self.assertFalse(soup.b.string) - - soup = self.soup("<b></b>") - self.assertFalse(soup.b.string) - - def test_all_text(self): - """Tag.text and Tag.get_text(sep=u"") -> all child text, concatenated""" - soup = self.soup("<a>a<b>r</b> <r> t </r></a>") - self.assertEqual(soup.a.text, "ar t ") - self.assertEqual(soup.a.get_text(strip=True), "art") - self.assertEqual(soup.a.get_text(","), "a,r, , t ") - self.assertEqual(soup.a.get_text(",", strip=True), "a,r,t") - - -class TestPersistence(SoupTest): - "Testing features like pickle and deepcopy." - - def setUp(self): - super(TestPersistence, self).setUp() - self.page = """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" -"http://www.w3.org/TR/REC-html40/transitional.dtd"> -<html> -<head> -<meta http-equiv="Content-Type" content="text/html; charset=utf-8"> -<title>Beautiful Soup: We called him Tortoise because he taught us.</title> -<link rev="made" href="mailto:leonardr@segfault.org"> -<meta name="Description" content="Beautiful Soup: an HTML parser optimized for screen-scraping."> -<meta name="generator" content="Markov Approximation 1.4 (module: leonardr)"> -<meta name="author" content="Leonard Richardson"> -</head> -<body> -<a href="foo">foo</a> -<a href="foo"><b>bar</b></a> -</body> -</html>""" - self.tree = self.soup(self.page) - - def test_pickle_and_unpickle_identity(self): - # Pickling a tree, then unpickling it, yields a tree identical - # to the original. - dumped = pickle.dumps(self.tree, 2) - loaded = pickle.loads(dumped) - self.assertEqual(loaded.__class__, BeautifulSoup) - self.assertEqual(loaded.decode(), self.tree.decode()) - - def test_deepcopy_identity(self): - # Making a deepcopy of a tree yields an identical tree. - copied = copy.deepcopy(self.tree) - self.assertEqual(copied.decode(), self.tree.decode()) - - def test_unicode_pickle(self): - # A tree containing Unicode characters can be pickled. - html = u"<b>\N{SNOWMAN}</b>" - soup = self.soup(html) - dumped = pickle.dumps(soup, pickle.HIGHEST_PROTOCOL) - loaded = pickle.loads(dumped) - self.assertEqual(loaded.decode(), soup.decode()) - - -class TestSubstitutions(SoupTest): - - def test_html_entity_substitution(self): - soup = self.soup( - u"<b>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</b>") - encoded = soup.encode("utf-8", substitute_html_entities=True) - self.assertEquals(encoded, - self.document_for("<b>Sacré bleu!</b>")) - - def test_html_entity_substitution_off_by_default(self): - markup = u"<b>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</b>" - soup = self.soup(markup) - encoded = soup.b.encode("utf-8") - self.assertEquals(encoded, markup.encode('utf-8')) - - def test_encoding_substitution(self): - # Here's the <meta> tag saying that a document is - # encoded in Shift-JIS. - meta_tag = ('<meta content="text/html; charset=x-sjis" ' - 'http-equiv="Content-type" />') - soup = self.soup(meta_tag) - - # Parse the document, and the charset is replaced with a - # generic value. - self.assertEquals(soup.meta['content'], - 'text/html; charset=%SOUP-ENCODING%') - - # Encode the document into some encoding, and the encoding is - # substituted into the meta tag. - utf_8 = soup.encode("utf-8") - self.assertTrue("charset=utf-8" in utf_8) - - euc_jp = soup.encode("euc_jp") - self.assertTrue("charset=euc_jp" in euc_jp) - - shift_jis = soup.encode("shift-jis") - self.assertTrue("charset=shift-jis" in shift_jis) - - utf_16_u = soup.encode("utf-16").decode("utf-16") - self.assertTrue("charset=utf-16" in utf_16_u) - - def test_encoding_substitution_doesnt_happen_if_tag_is_strained(self): - markup = ('<head><meta content="text/html; charset=x-sjis" ' - 'http-equiv="Content-type" /></head><pre>foo</pre>') - - # Beautiful Soup used to try to rewrite the meta tag even if the - # meta tag got filtered out by the strainer. This test makes - # sure that doesn't happen. - strainer = SoupStrainer('pre') - soup = self.soup(markup, parse_only=strainer) - self.assertEquals(soup.contents[0].name, 'pre') - - -class TestEncoding(SoupTest): - """Test the ability to encode objects into strings.""" - - def test_unicode_string_can_be_encoded(self): - html = u"<b>\N{SNOWMAN}</b>" - soup = self.soup(html) - self.assertEquals(soup.b.string.encode("utf-8"), - u"\N{SNOWMAN}".encode("utf-8")) - - def test_tag_containing_unicode_string_can_be_encoded(self): - html = u"<b>\N{SNOWMAN}</b>" - soup = self.soup(html) - self.assertEquals( - soup.b.encode("utf-8"), html.encode("utf-8")) - - -class TestNavigableStringSubclasses(SoupTest): - - def test_cdata(self): - # None of the current builders turn CDATA sections into CData - # objects, but you can create them manually. - soup = self.soup("") - cdata = CData("foo") - soup.insert(1, cdata) - self.assertEquals(str(soup), "<![CDATA[foo]]>") - self.assertEquals(soup.find(text="foo"), "foo") - self.assertEquals(soup.contents[0], "foo") |