diff options
| -rw-r--r-- | MANIFEST.in | 2 | ||||
| -rw-r--r-- | NEWS.txt | 15 | ||||
| -rw-r--r-- | bs4/testing.py | 320 | ||||
| -rw-r--r-- | bs4/tests/test_html5lib.py | 324 | ||||
| -rw-r--r-- | bs4/tests/test_htmlparser.py | 573 | ||||
| -rw-r--r-- | bs4/tests/test_lxml.py | 234 | ||||
| -rw-r--r-- | bs4/tests/test_soup.py | 45 | ||||
| -rw-r--r-- | scripts/demonstrate_parser_differences.py | 95 | ||||
| -rw-r--r-- | scripts/demonstration_markup.txt | 34 |
9 files changed, 534 insertions, 1108 deletions
diff --git a/MANIFEST.in b/MANIFEST.in index fe99812..b5504e6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,5 @@ include doc/Makefile include doc/source/*.py include doc/source/*.rst include doc/source/*.jpg +include scripts/*.py +include scripts/*.txt @@ -1,4 +1,4 @@ -= 4.0.0b7 () = += 4.0.0b7 (20110223) = * Upon decoding to string, any characters that can't be represented in your chosen encoding will be converted into numeric XML entity @@ -14,6 +14,19 @@ * It's now possible to deepcopy a BeautifulSoup object created with Python's built-in HTML parser. +* About 100 unit tests that "test" the behavior of various parsers on + invalid markup have been removed. Legitimate changes to those + parsers caused these tests to fail, indicating that perhaps + Beautiful Soup should not test the behavior of foreign + libraries. + + The problematic unit tests have been reformulated as informational + comparisons generated by the script + scripts/demonstrate_parser_differences.py. + + This makes Beautiful Soup compatible with html5lib version 0.95 and + future versions of HTMLParser. + = 4.0.0b6 (20110216) = * Multi-valued attributes like "class" always have a list of values, diff --git a/bs4/testing.py b/bs4/testing.py index e5e8c93..cc30e17 100644 --- a/bs4/testing.py +++ b/bs4/testing.py @@ -1,16 +1,19 @@ """Helper classes for tests.""" +import copy import functools import unittest from unittest import TestCase from bs4 import BeautifulSoup -from bs4.element import Comment, SoupStrainer -try: - from bs4.builder import LXMLTreeBuilder - default_builder = LXMLTreeBuilder -except ImportError, e: - from bs4.builder import HTMLParserTreeBuilder - default_builder = HTMLParserTreeBuilder +from bs4.element import ( + Comment, + Doctype, + SoupStrainer, +) + +from bs4.builder import HTMLParserTreeBuilder +default_builder = HTMLParserTreeBuilder + class SoupTest(unittest.TestCase): @@ -38,6 +41,309 @@ class SoupTest(unittest.TestCase): self.assertEqual(obj.decode(), self.document_for(compare_parsed_to)) + +class HTMLTreeBuilderSmokeTest(object): + + """A basic test of a treebuilder's competence. + + Any HTML treebuilder, present or future, should be able to pass + these tests. With invalid markup, there's room for interpretation, + and different parsers can handle it differently. But with the + markup in these tests, there's not much room for interpretation. + """ + + def assertDoctypeHandled(self, doctype_fragment): + """Assert that a given doctype string is handled correctly.""" + doctype_str, soup = self._document_with_doctype(doctype_fragment) + + # Make sure a Doctype object was created. + doctype = soup.contents[0] + self.assertEqual(doctype.__class__, Doctype) + self.assertEqual(doctype, doctype_fragment) + self.assertEqual(str(soup)[:len(doctype_str)], doctype_str) + + # Make sure that the doctype was correctly associated with the + # parse tree and that the rest of the document parsed. + self.assertEqual(soup.p.contents[0], 'foo') + + def _document_with_doctype(self, doctype_fragment): + """Generate and parse a document with the given doctype.""" + doctype = '<!DOCTYPE %s>' % doctype_fragment + markup = doctype + '\n<p>foo</p>' + soup = self.soup(markup) + return doctype, soup + + def test_normal_doctypes(self): + """Make sure normal, everyday HTML doctypes are handled correctly.""" + self.assertDoctypeHandled("html") + self.assertDoctypeHandled( + 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"') + + def test_namespaced_system_doctype(self): + # We can handle a namespaced doctype with a system ID. + self.assertDoctypeHandled('xsl:stylesheet SYSTEM "htmlent.dtd"') + + def test_namespaced_public_doctype(self): + # Test a namespaced doctype with a public id. + self.assertDoctypeHandled('xsl:stylesheet PUBLIC "htmlent.dtd"') + + def test_deepcopy(self): + """Make sure you can copy the tree builder. + + This is important because the builder is part of a + BeautifulSoup object, and we want to be able to copy that. + """ + copy.deepcopy(self.default_builder) + + def test_p_tag_is_never_empty_element(self): + """A <p> tag is never designated as an empty-element tag. + + Even if the markup shows it as an empty-element tag, it + shouldn't be presented that way. + """ + soup = self.soup("<p/>") + self.assertFalse(soup.p.is_empty_element) + self.assertEqual(str(soup.p), "<p></p>") + + def test_unclosed_tags_get_closed(self): + """A tag that's not closed by the end of the document should be closed. + + This applies to all tags except empty-element tags. + """ + self.assertSoupEquals("<p>", "<p></p>") + self.assertSoupEquals("<b>", "<b></b>") + + self.assertSoupEquals("<br>", "<br/>") + + def test_br_is_always_empty_element_tag(self): + """A <br> tag is designated as an empty-element tag. + + Some parsers treat <br></br> as one <br/> tag, some parsers as + two tags, but it should always be an empty-element tag. + """ + soup = self.soup("<br></br>") + self.assertTrue(soup.br.is_empty_element) + self.assertEqual(str(soup.br), "<br/>") + + def test_comment(self): + # Comments are represented as Comment objects. + markup = "<p>foo<!--foobar-->baz</p>" + self.assertSoupEquals(markup) + + soup = self.soup(markup) + comment = soup.find(text="foobar") + self.assertEqual(comment.__class__, Comment) + + def test_preserved_whitespace_in_pre_and_textarea(self): + """Whitespace must be preserved in <pre> and <textarea> tags.""" + self.assertSoupEquals("<pre> </pre>") + self.assertSoupEquals("<textarea> woo </textarea>") + + def test_nested_inline_elements(self): + """Inline elements can be nested indefinitely.""" + b_tag = "<b>Inside a B tag</b>" + self.assertSoupEquals(b_tag) + + nested_b_tag = "<p>A <i>nested <b>tag</b></i></p>" + self.assertSoupEquals(nested_b_tag) + + double_nested_b_tag = "<p>A <a>doubly <i>nested <b>tag</b></i></a></p>" + self.assertSoupEquals(nested_b_tag) + + def test_nested_block_level_elements(self): + """Block elements can be nested.""" + soup = self.soup('<blockquote><p><b>Foo</b></p></blockquote>') + blockquote = soup.blockquote + self.assertEqual(blockquote.p.b.string, 'Foo') + self.assertEqual(blockquote.b.string, 'Foo') + + def test_correctly_nested_tables(self): + """One table can go inside another one.""" + markup = ('<table id="1">' + '<tr>' + "<td>Here's another table:" + '<table id="2">' + '<tr><td>foo</td></tr>' + '</table></td>') + + self.assertSoupEquals( + markup, + '<table id="1"><tr><td>Here\'s another table:' + '<table id="2"><tr><td>foo</td></tr></table>' + '</td></tr></table>') + + self.assertSoupEquals( + "<table><thead><tr><td>Foo</td></tr></thead>" + "<tbody><tr><td>Bar</td></tr></tbody>" + "<tfoot><tr><td>Baz</td></tr></tfoot></table>") + + def test_angle_brackets_in_attribute_values_are_escaped(self): + self.assertSoupEquals('<a b="<a>"></a>', '<a b="<a>"></a>') + + def test_entities_in_attributes_converted_to_unicode(self): + expect = u'<p id="pi\N{LATIN SMALL LETTER N WITH TILDE}ata"></p>' + self.assertSoupEquals('<p id="piñata"></p>', expect) + self.assertSoupEquals('<p id="piñata"></p>', expect) + self.assertSoupEquals('<p id="piñata"></p>', expect) + + def test_entities_in_text_converted_to_unicode(self): + expect = u'<p>pi\N{LATIN SMALL LETTER N WITH TILDE}ata</p>' + self.assertSoupEquals("<p>piñata</p>", expect) + self.assertSoupEquals("<p>piñata</p>", expect) + self.assertSoupEquals("<p>piñata</p>", expect) + + def test_out_of_range_entity(self): + expect = u"\N{REPLACEMENT CHARACTER}" + self.assertSoupEquals("�", expect) + self.assertSoupEquals("�", expect) + self.assertSoupEquals("�", expect) + + # + # Generally speaking, tests below this point are more tests of + # Beautiful Soup than tests of the tree builders. But parsers are + # weird, so we run these tests separately for every tree builder + # to detect any differences between them. + # + + def test_soupstrainer(self): + """Parsers should be able to work with SoupStrainers.""" + strainer = SoupStrainer("b") + soup = self.soup("A <b>bold</b> <meta/> <i>statement</i>", + parse_only=strainer) + self.assertEqual(soup.decode(), "<b>bold</b>") + + def test_single_quote_attribute_values_become_double_quotes(self): + self.assertSoupEquals("<foo attr='bar'></foo>", + '<foo attr="bar"></foo>') + + def test_attribute_values_with_nested_quotes_are_left_alone(self): + text = """<foo attr='bar "brawls" happen'>a</foo>""" + self.assertSoupEquals(text) + + def test_attribute_values_with_double_nested_quotes_get_quoted(self): + text = """<foo attr='bar "brawls" happen'>a</foo>""" + soup = self.soup(text) + soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"' + self.assertSoupEquals( + soup.foo.decode(), + """<foo attr="Brawls happen at "Bob\'s Bar"">a</foo>""") + + def test_ampersand_in_attribute_value_gets_escaped(self): + self.assertSoupEquals('<this is="really messed up & stuff"></this>', + '<this is="really messed up & stuff"></this>') + + self.assertSoupEquals( + '<a href="http://example.org?a=1&b=2;3">foo</a>', + '<a href="http://example.org?a=1&b=2;3">foo</a>') + + def test_escaped_ampersand_in_attribute_value_is_left_alone(self): + self.assertSoupEquals('<a href="http://example.org?a=1&b=2;3"></a>') + + def test_entities_in_strings_converted_during_parsing(self): + # Both XML and HTML entities are converted to Unicode characters + # during parsing. + text = "<p><<sacré bleu!>></p>" + expected = u"<p><<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></p>" + self.assertSoupEquals(text, expected) + + def test_smart_quotes_converted_on_the_way_in(self): + # Microsoft smart quotes are converted to Unicode characters during + # parsing. + quote = b"<p>\x91Foo\x92</p>" + soup = self.soup(quote) + self.assertEqual( + soup.p.string, + u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}") + + def test_non_breaking_spaces_converted_on_the_way_in(self): + soup = self.soup("<a> </a>") + self.assertEqual(soup.a.string, u"\N{NO-BREAK SPACE}" * 2) + + def test_entities_converted_on_the_way_out(self): + text = "<p><<sacré bleu!>></p>" + expected = u"<p><<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></p>".encode("utf-8") + soup = self.soup(text) + self.assertEqual(soup.p.encode("utf-8"), expected) + + def test_real_iso_latin_document(self): + # Smoke test of interrelated functionality, using an + # easy-to-understand document. + + # Here it is in Unicode. Note that it claims to be in ISO-Latin-1. + unicode_html = u'<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>' + + # That's because we're going to encode it into ISO-Latin-1, and use + # that to test. + iso_latin_html = unicode_html.encode("iso-8859-1") + + # Parse the ISO-Latin-1 HTML. + soup = self.soup(iso_latin_html) + # Encode it to UTF-8. + result = soup.encode("utf-8") + + # What do we expect the result to look like? Well, it would + # look like unicode_html, except that the META tag would say + # UTF-8 instead of ISO-Latin-1. + expected = unicode_html.replace("ISO-Latin-1", "utf-8") + + # And, of course, it would be in UTF-8, not Unicode. + expected = expected.encode("utf-8") + + # Ta-da! + self.assertEqual(result, expected) + + def test_real_shift_jis_document(self): + # Smoke test to make sure the parser can handle a document in + # Shift-JIS encoding, without choking. + shift_jis_html = ( + b'<html><head></head><body><pre>' + b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f' + b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c' + b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B' + b'</pre></body></html>') + unicode_html = shift_jis_html.decode("shift-jis") + soup = self.soup(unicode_html) + + # Make sure the parse tree is correctly encoded to various + # encodings. + self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8")) + self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp")) + + def test_real_hebrew_document(self): + # A real-world test to make sure we can convert ISO-8859-9 (a + # Hebrew encoding) to UTF-8. + hebrew_document = b'<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>' + soup = self.soup( + hebrew_document, from_encoding="iso8859-8") + self.assertEqual(soup.original_encoding, 'iso8859-8') + self.assertEqual( + soup.encode('utf-8'), + hebrew_document.decode("iso8859-8").encode("utf-8")) + + def test_meta_tag_reflects_current_encoding(self): + # Here's the <meta> tag saying that a document is + # encoded in Shift-JIS. + meta_tag = ('<meta content="text/html; charset=x-sjis" ' + 'http-equiv="Content-type"/>') + + # Here's a document incorporating that meta tag. + shift_jis_html = ( + '<html><head>\n%s\n' + '<meta http-equiv="Content-language" content="ja"/>' + '</head><body>Shift-JIS markup goes here.') % meta_tag + soup = self.soup(shift_jis_html) + + # Parse the document, and the charset is replaced with a + # generic value. + parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'}) + self.assertEqual(parsed_meta['content'], + 'text/html; charset=%SOUP-ENCODING%') + self.assertEqual(parsed_meta.contains_substitutions, True) + + # For the rest of the story, see TestSubstitutions in + # test_tree.py. + + def skipIf(condition, reason): def nothing(test, *args, **kwargs): return None diff --git a/bs4/tests/test_html5lib.py b/bs4/tests/test_html5lib.py index 3f00d52..f1edddf 100644 --- a/bs4/tests/test_html5lib.py +++ b/bs4/tests/test_html5lib.py @@ -1,18 +1,22 @@ +"""Tests to ensure that the html5lib tree builder generates good trees.""" + try: from bs4.builder import HTML5TreeBuilder HTML5LIB_PRESENT = True except ImportError, e: HTML5LIB_PRESENT = False -from bs4.element import Comment, SoupStrainer -import test_htmlparser -import unittest -from bs4.testing import skipIf +from bs4.element import SoupStrainer +from bs4.testing import ( + HTMLTreeBuilderSmokeTest, + SoupTest, + skipIf, +) @skipIf( not HTML5LIB_PRESENT, "html5lib seems not to be present, not testing its tree builder.") -class TestHTML5Builder(test_htmlparser.TestHTMLParserTreeBuilder): - """See `BuilderSmokeTest`.""" +class HTML5LibBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest): + """See ``HTMLTreeBuilderSmokeTest``.""" @property def default_builder(self): @@ -27,15 +31,8 @@ class TestHTML5Builder(test_htmlparser.TestHTMLParserTreeBuilder): self.assertEqual( soup.decode(), self.document_for(markup)) - def test_bare_string(self): - # A bare string is turned into some kind of HTML document or - # fragment recognizable as the original string. - # - # In this case, html5lib puts a <p> tag around the bare string. - self.assertSoupEquals( - "A bare string", "A bare string") - def test_correctly_nested_tables(self): + """html5lib inserts <tbody> tags where other parsers don't.""" markup = ('<table id="1">' '<tr>' "<td>Here's another table:" @@ -53,302 +50,3 @@ class TestHTML5Builder(test_htmlparser.TestHTMLParserTreeBuilder): "<table><thead><tr><td>Foo</td></tr></thead>" "<tbody><tr><td>Bar</td></tr></tbody>" "<tfoot><tr><td>Baz</td></tr></tfoot></table>") - - def test_literal_in_textarea(self): - markup = '<textarea>Junk like <b> tags and <&<&</textarea>' - soup = self.soup(markup) - self.assertEqual( - soup.textarea.contents, ["Junk like <b> tags and <&<&"]) - - def test_collapsed_whitespace(self): - """Whitespace is preserved even in tags that don't require it.""" - self.assertSoupEquals("<p> </p>") - self.assertSoupEquals("<b> </b>") - - def test_cdata_where_its_ok(self): - # In html5lib 0.9.0, all CDATA sections are converted into - # comments. In a later version (unreleased as of this - # writing), CDATA sections in tags like <svg> and <math> will - # be preserved. BUT, I'm not sure how Beautiful Soup needs to - # adjust to transform this preservation into the construction - # of a BS CData object. - markup = "<svg><![CDATA[foobar]]>" - - # Eventually we should be able to do a find(text="foobar") and - # get a CData object. - self.assertSoupEquals(markup, "<svg><!--[CDATA[foobar]]--></svg>") - - def test_entities_in_attribute_values_converted_during_parsing(self): - - # The numeric entity is recognized even without the closing - # semicolon. - text = '<x t="piñata">' - expected = u"pi\N{LATIN SMALL LETTER N WITH TILDE}ata" - soup = self.soup(text) - self.assertEqual(soup.x['t'], expected) - - def test_naked_ampersands(self): - # Ampersands are not treated as entities, unlike in html.parser. - text = "<p>AT&T</p>" - soup = self.soup(text) - self.assertEqual(soup.p.string, "AT&T") - - def test_namespaced_system_doctype(self): - # Test a namespaced doctype with a system id. - self._test_doctype('xsl:stylesheet SYSTEM "htmlent.dtd"') - - def test_namespaced_public_doctype(self): - # Test a namespaced doctype with a public id. - self._test_doctype('xsl:stylesheet PUBLIC "htmlent.dtd"') - - -@skipIf( - not HTML5LIB_PRESENT, - "html5lib seems not to be present, not testing it on invalid markup.") -class TestHTML5BuilderInvalidMarkup( - test_htmlparser.TestHTMLParserTreeBuilderInvalidMarkup): - """See `BuilderInvalidMarkupSmokeTest`.""" - - @property - def default_builder(self): - return HTML5TreeBuilder() - - def test_unclosed_block_level_elements(self): - # The unclosed <b> tag is closed so that the block-level tag - # can be closed, and another <b> tag is inserted after the - # next block-level tag begins. - self.assertSoupEquals( - '<blockquote><p><b>Foo</blockquote><p>Bar', - '<blockquote><p><b>Foo</b></p></blockquote><p><b>Bar</b></p>') - - def test_attribute_value_never_got_closed(self): - markup = '<a href="http://foo.com/</a> and blah and blah' - soup = self.soup(markup) - self.assertEqual( - soup.a['href'], "http://foo.com/</a> and blah and blah") - - def test_attribute_value_was_closed_by_subsequent_tag(self): - markup = """<a href="foo</a>, </a><a href="bar">baz</a>""" - soup = self.soup(markup) - # The string between the first and second quotes was interpreted - # as the value of the 'href' attribute. - self.assertEqual(soup.a['href'], 'foo</a>, </a><a href=') - - #The string after the second quote (bar"), was treated as an - #empty attribute called bar. - self.assertEqual(soup.a['bar'], '') - self.assertEqual(soup.a.string, "baz") - - def test_document_starts_with_bogus_declaration(self): - soup = self.soup('<! Foo ><p>a</p>') - # The declaration is ignored altogether. - self.assertEqual(soup.encode(), b"<html><body><p>a</p></body></html>") - - def test_table_containing_bare_markup(self): - # Markup should be in table cells, not directly in the table. - self.assertSoupEquals("<table><div>Foo</div></table>", - "<div>Foo</div><table></table>") - - def test_unclosed_a_tag(self): - # n.b. the whitespace is important here. - markup = """<div id="1"> - <a href="foo"> -</div> -<div id="2"> - <div id="3"> - <a href="bar"></a> - </div> -</div>""" - - expect = """<div id="1"> - <a href="foo"> -</a></div><a href="foo"> -</a><div id="2"><a href="foo"> - </a><div id="3"><a href="foo"> - </a><a href="bar"></a> - </div> -</div>""" - self.assertSoupEquals(markup, expect) - - def test_incorrectly_nested_tables(self): - self.assertSoupEquals( - '<table><tr><table><tr id="nested">', - ('<table><tbody><tr></tr></tbody></table>' - '<table><tbody><tr id="nested"></tr></tbody></table>')) - - def test_floating_text_in_table(self): - self.assertSoupEquals( - "<table><td></td>foo<td>bar</td></table>", - "foo<table><tbody><tr><td></td><td>bar</td></tr></tbody></table>") - - def test_empty_element_tag_with_contents(self): - self.assertSoupEquals("<br>foo</br>", "<br/>foo<br/>") - - def test_doctype_in_body(self): - markup = "<p>one<!DOCTYPE foobar>two</p>" - self.assertSoupEquals(markup, "<p>onetwo</p>") - - def test_cdata_where_it_doesnt_belong(self): - # Random CDATA sections are converted into comments. - markup = "<div><![CDATA[foo]]>" - soup = self.soup(markup) - data = soup.find(text="[CDATA[foo]]") - self.assertEqual(data.__class__, Comment) - - def test_nonsensical_declaration(self): - # Declarations that don't make any sense are turned into comments. - soup = self.soup('<! Foo = -8><p>a</p>') - self.assertEqual(str(soup), - ("<!-- Foo = -8-->" - "<html><head></head><body><p>a</p></body></html>")) - - soup = self.soup('<p>a</p><! Foo = -8>') - self.assertEqual(str(soup), - ("<html><head></head><body><p>a</p>" - "<!-- Foo = -8--></body></html>")) - - def test_whitespace_in_doctype(self): - # A declaration that has extra whitespace is turned into a comment. - soup = self.soup(( - '<! DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">' - '<p>foo</p>')) - self.assertEqual( - str(soup), - ('<!-- DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"-->' - '<html><head></head><body><p>foo</p></body></html>')) - - def test_incomplete_declaration(self): - # An incomplete declaration is treated as a comment. - markup = 'a<!b <p>c' - self.assertSoupEquals(markup, "a<!--b <p-->c") - - # Let's spell that out a little more explicitly. - soup = self.soup(markup) - str1, comment, str2 = soup.body.contents - self.assertEqual(str1, 'a') - self.assertEqual(comment.__class__, Comment) - self.assertEqual(comment, 'b <p') - self.assertEqual(str2, 'c') - - def test_document_starts_with_bogus_declaration(self): - soup = self.soup('<! Foo >a') - # 'Foo' becomes a comment that appears before the HTML. - comment = soup.contents[0] - self.assertTrue(isinstance(comment, Comment)) - self.assertEqual(comment, 'Foo') - - self.assertEqual(self.find(text="a") == "a") - - def test_attribute_value_was_closed_by_subsequent_tag(self): - markup = """<a href="foo</a>, </a><a href="bar">baz</a>""" - soup = self.soup(markup) - # The string between the first and second quotes was interpreted - # as the value of the 'href' attribute. - self.assertEqual(soup.a['href'], 'foo</a>, </a><a href=') - - #The string after the second quote (bar"), was treated as an - #empty attribute called bar". - self.assertEqual(soup.a['bar"'], '') - self.assertEqual(soup.a.string, "baz") - - def test_document_starts_with_bogus_declaration(self): - soup = self.soup('<! Foo ><p>a</p>') - # The declaration becomes a comment. - comment = soup.contents[0] - self.assertTrue(isinstance(comment, Comment)) - self.assertEqual(comment, ' Foo ') - self.assertEqual(soup.p.string, 'a') - - def test_document_ends_with_incomplete_declaration(self): - soup = self.soup('<p>a<!b') - # This becomes a string 'a'. The incomplete declaration is ignored. - # Compare html5lib, which turns it into a comment. - s, comment = soup.p.contents - self.assertEqual(s, 'a') - self.assertTrue(isinstance(comment, Comment)) - self.assertEqual(comment, 'b') - - def test_entity_was_not_finished(self): - soup = self.soup("<p><Hello>") - # Compare html5lib, which completes the entity. - self.assertEqual(soup.p.string, "<Hello>") - - def test_nonexistent_entity(self): - soup = self.soup("<p>foo&#bar;baz</p>") - self.assertEqual(soup.p.string, "foo&#bar;baz") - - # Compare a real entity. - soup = self.soup("<p>foodbaz</p>") - self.assertEqual(soup.p.string, "foodbaz") - - def test_entity_out_of_range(self): - # An entity that's out of range will be converted to - # REPLACEMENT CHARACTER. - soup = self.soup("<p>�</p>") - self.assertEqual(soup.p.string, u"\N{REPLACEMENT CHARACTER}") - - soup = self.soup("<p>�</p>") - self.assertEqual(soup.p.string, u"\N{REPLACEMENT CHARACTER}") - - def test_incomplete_declaration(self): - self.assertSoupEquals('a<!b <p>c', 'a<!--b <p-->c') - - def test_nonsensical_declaration(self): - soup = self.soup('<! Foo = -8><p>a</p>') - self.assertEquals( - soup.decode(), - "<!-- Foo = -8--><html><head></head><body><p>a</p></body></html>") - - def test_unquoted_attribute_value(self): - soup = self.soup('<a style={height:21px;}></a>') - self.assertEqual(soup.a['style'], '{height:21px;}') - - def test_boolean_attribute_with_no_value(self): - soup = self.soup("<table><td nowrap>foo</td></table>") - self.assertEqual(soup.table.td['nowrap'], '') - - def test_cdata_where_it_doesnt_belong(self): - #CDATA sections are ignored. - markup = "<div><![CDATA[foo]]>" - self.assertSoupEquals(markup, "<div><!--[CDATA[foo]]--></div>") - - def test_empty_element_tag_with_contents(self): - self.assertSoupEquals("<br>foo</br>", "<br/>foo<br/>") - - def test_fake_self_closing_tag(self): - # If a self-closing tag presents as a normal tag, the 'open' - # tag is treated as an instance of the self-closing tag and - # the 'close' tag is ignored. - self.assertSoupEquals( - "<item><link>http://foo.com/</link></item>", - "<item><link/>http://foo.com/</item>") - - def test_paragraphs_containing_block_display_elements(self): - markup = self.soup("<p>this is the definition:" - "<dl><dt>first case</dt>") - # The <p> tag is closed before the <dl> tag begins. - self.assertEqual(markup.p.contents, ["this is the definition:"]) - - def test_multiple_values_for_the_same_attribute(self): - markup = '<b b="20" a="1" b="10" a="2" a="3" a="4"></b>' - self.assertSoupEquals(markup, '<b a="1" b="20"></b>') - - -@skipIf( - not HTML5LIB_PRESENT, - "html5lib seems not to be present, not testing it on encoding conversion.") -class TestHTML5LibEncodingConversion( - test_htmlparser.TestHTMLParserTreeBuilderEncodingConversion): - @property - def default_builder(self): - return HTML5TreeBuilder() - - def test_real_hebrew_document(self): - # A real-world test to make sure we can convert ISO-8859-8 (a - # Hebrew encoding) to UTF-8. - soup = self.soup(self.HEBREW_DOCUMENT, - from_encoding="iso-8859-8") - self.assertEqual(soup.original_encoding, 'iso8859-8') - self.assertEqual( - soup.encode('utf-8'), - self.HEBREW_DOCUMENT.decode("iso-8859-8").encode("utf-8")) diff --git a/bs4/tests/test_htmlparser.py b/bs4/tests/test_htmlparser.py index d5b6ae1..bcb5ed2 100644 --- a/bs4/tests/test_htmlparser.py +++ b/bs4/tests/test_htmlparser.py @@ -1,574 +1,19 @@ -import copy -from HTMLParser import HTMLParseError -from bs4.element import Comment, Doctype, SoupStrainer -from bs4.builder import HTMLParserTreeBuilder -from bs4.element import CData -from bs4.testing import SoupTest - -class TestHTMLParserTreeBuilder(SoupTest): +"""Tests to ensure that the html.parser tree builder generates good +trees.""" - """A smoke test for the built-in tree builder. - - Subclass this to test some other HTML tree builder. Subclasses of - this test ensure that all of Beautiful Soup's tree builders - generate more or less the same trees. +from bs4.testing import SoupTest, HTMLTreeBuilderSmokeTest +from bs4.builder import HTMLParserTreeBuilder - It's okay for trees to differ--just override the appropriate test - method to demonstrate how one tree builder differs from the - default builder. But in general, all HTML tree builders should - generate trees that make most of these tests pass. - """ +class HTMLParserTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest): @property def default_builder(self): return HTMLParserTreeBuilder() - def test_bare_string(self): - # A bare string is turned into some kind of HTML document or - # fragment recognizable as the original string. - # - # HTMLParser does not modify the bare string at all. - self.assertSoupEquals("A bare string") - - def test_cdata_where_its_ok(self): - # HTMLParser recognizes CDATA sections and passes them through. - markup = "<svg><![CDATA[foobar]]></svg>" - self.assertSoupEquals(markup) - soup = self.soup(markup) - string = soup.svg.string - self.assertEqual(string, "foobar") - self.assertTrue(isinstance(string, CData)) - - def test_hex_entities_in_text(self): - # XXX This tests a workaround for a bug in HTMLParser. - self.assertSoupEquals("<p>ñ</p>", u"<p>\xf1</p>") - - def test_entities_in_attribute_values_converted_during_parsing(self): - - # The numeric entity isn't recognized without the closing - # semicolon. - text = '<x t="piñata">' - expected = u"pi\N{LATIN SMALL LETTER N WITH TILDE}ata" - soup = self.soup(text) - self.assertEqual(soup.x['t'], "piñata") - - text = '<x t="piñata">' - expected = u"pi\N{LATIN SMALL LETTER N WITH TILDE}ata" - soup = self.soup(text) - self.assertEqual(soup.x['t'], u"pi\xf1ata") - - text = '<x t="piñata">' - soup = self.soup(text) - self.assertEqual(soup.x['t'], expected) - - text = '<x t="sacré bleu">' - soup = self.soup(text) - self.assertEqual( - soup.x['t'], - u"sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu") - - # This can cause valid HTML to become invalid. - valid_url = '<a href="http://example.org?a=1&b=2;3">foo</a>' - soup = self.soup(valid_url) - self.assertEqual(soup.a['href'], "http://example.org?a=1&b=2;3") - - # I think it would be very difficult to 'fix' these tests, judging - # from my experience with previous versions of Beautiful Soup. - def test_naked_ampersands(self): - # Ampersands are treated as entities. - text = "<p>AT&T</p>" - soup = self.soup(text) - self.assertEqual(soup.p.string, "AT&T;") - - def test_literal_in_textarea(self): - # Anything inside a <textarea> is supposed to be treated as - # the literal value of the field, (XXX citation - # needed). html5lib does this correctly. But, HTMLParser does its - # best to parse the contents of a <textarea> as HTML. - text = '<textarea>Junk like <b> tags and <&<&</textarea>' - soup = self.soup(text) - self.assertEqual(len(soup.textarea.contents), 2) - self.assertEqual(soup.textarea.contents[0], u"Junk like ") - self.assertEqual(soup.textarea.contents[1].name, 'b') - self.assertEqual(soup.textarea.b.string, u" tags and <&<&") - - def test_literal_in_script(self): - # Some versions of HTMLParser choke on markup like this: - # if (i < 2) { alert("<b>foo</b>"); } - # Some versions of HTMLParser don't. - # - # The easiest thing is to just not run this test for HTMLParser. - pass - - # Namespaced doctypes cause an HTMLParseError def test_namespaced_system_doctype(self): - self.assertRaises(HTMLParseError, self._test_doctype, - 'xsl:stylesheet SYSTEM "htmlent.dtd"') + # html.parser can't handle namespaced doctypes, so skip this one. + pass def test_namespaced_public_doctype(self): - self.assertRaises(HTMLParseError, self._test_doctype, - 'xsl:stylesheet PUBLIC "htmlent.dtd"') - - def _test_doctype(self, doctype_fragment): - """Run a battery of assertions on a given doctype string. - - HTMLParser doesn't actually behave like this, so this method - is never called in this class. But many other builders do - behave like this, so I've put the method in the superclass. - """ - doctype_str = '<!DOCTYPE %s>' % doctype_fragment - markup = doctype_str + '<p>foo</p>' - soup = self.soup(markup) - doctype = soup.contents[0] - self.assertEqual(doctype.__class__, Doctype) - self.assertEqual(doctype, doctype_fragment) - self.assertEqual(str(soup)[:len(doctype_str)], doctype_str) - - # Make sure that the doctype was correctly associated with the - # parse tree and that the rest of the document parsed. - self.assertEqual(soup.p.contents[0], 'foo') - -# ------------------------- - - def test_mixed_case_tags(self): - # Mixed-case tags are folded to lowercase. - self.assertSoupEquals( - "<a><B><Cd><EFG></efg></CD></b></A>", - "<a><b><cd><efg></efg></cd></b></a>") - - - def test_empty_tag_thats_not_an_empty_element_tag(self): - # A tag that is empty but not an HTML empty-element tag - # is not presented as an empty-element tag. - self.assertSoupEquals("<p>", "<p></p>") - - def test_comment(self): - # Comments are represented as Comment objects. - markup = "<p>foo<!--foobar-->baz</p>" - self.assertSoupEquals(markup) - - soup = self.soup(markup) - comment = soup.find(text="foobar") - self.assertEqual(comment.__class__, Comment) - - def test_nested_inline_elements(self): - # Inline tags can be nested indefinitely. - b_tag = "<b>Inside a B tag</b>" - self.assertSoupEquals(b_tag) - - nested_b_tag = "<p>A <i>nested <b>tag</b></i></p>" - self.assertSoupEquals(nested_b_tag) - - double_nested_b_tag = "<p>A <a>doubly <i>nested <b>tag</b></i></a></p>" - self.assertSoupEquals(nested_b_tag) - - def test_nested_block_level_elements(self): - soup = self.soup('<blockquote><p><b>Foo</b></p></blockquote>') - blockquote = soup.blockquote - self.assertEqual(blockquote.p.b.string, 'Foo') - self.assertEqual(blockquote.b.string, 'Foo') - - # This is a <table> tag containing another <table> tag in one of its - # cells. - TABLE_MARKUP_1 = ('<table id="1">' - '<tr>' - "<td>Here's another table:" - '<table id="2">' - '<tr><td>foo</td></tr>' - '</table></td>') - - def test_correctly_nested_tables(self): - markup = ('<table id="1">' - '<tr>' - "<td>Here's another table:" - '<table id="2">' - '<tr><td>foo</td></tr>' - '</table></td>') - - self.assertSoupEquals( - markup, - '<table id="1"><tr><td>Here\'s another table:' - '<table id="2"><tr><td>foo</td></tr></table>' - '</td></tr></table>') - - self.assertSoupEquals( - "<table><thead><tr><td>Foo</td></tr></thead>" - "<tbody><tr><td>Bar</td></tr></tbody>" - "<tfoot><tr><td>Baz</td></tr></tfoot></table>") - - def test_collapsed_whitespace(self): - """In most tags, whitespace is collapsed.""" - self.assertSoupEquals("<p> </p>", "<p> </p>") - - def test_preserved_whitespace_in_pre_and_textarea(self): - """In <pre> and <textarea> tags, whitespace is preserved.""" - self.assertSoupEquals("<pre> </pre>") - self.assertSoupEquals("<textarea> woo </textarea>") - - def test_single_quote_attribute_values_become_double_quotes(self): - self.assertSoupEquals("<foo attr='bar'></foo>", - '<foo attr="bar"></foo>') - - def test_attribute_values_with_nested_quotes_are_left_alone(self): - text = """<foo attr='bar "brawls" happen'>a</foo>""" - self.assertSoupEquals(text) - - def test_attribute_values_with_double_nested_quotes_get_quoted(self): - text = """<foo attr='bar "brawls" happen'>a</foo>""" - soup = self.soup(text) - soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"' - self.assertSoupEquals( - soup.foo.decode(), - """<foo attr="Brawls happen at "Bob\'s Bar"">a</foo>""") - - def test_ampersand_in_attribute_value_gets_quoted(self): - self.assertSoupEquals('<this is="really messed up & stuff"></this>', - '<this is="really messed up & stuff"></this>') - - def test_entities_in_strings_converted_during_parsing(self): - # Both XML and HTML entities are converted to Unicode characters - # during parsing. - text = "<p><<sacré bleu!>></p>" - expected = u"<p><<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></p>" - self.assertSoupEquals(text, expected) - - def test_smart_quotes_converted_on_the_way_in(self): - # Microsoft smart quotes are converted to Unicode characters during - # parsing. - quote = b"<p>\x91Foo\x92</p>" - soup = self.soup(quote) - self.assertEqual( - soup.p.string, - u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}") - - def test_non_breaking_spaces_converted_on_the_way_in(self): - soup = self.soup("<a> </a>") - self.assertEqual(soup.a.string, u"\N{NO-BREAK SPACE}" * 2) - - def test_real_iso_latin_document(self): - # Smoke test of interrelated functionality, using an - # easy-to-understand document. - - # Here it is in Unicode. Note that it claims to be in ISO-Latin-1. - unicode_html = u'<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>' - - # That's because we're going to encode it into ISO-Latin-1, and use - # that to test. - iso_latin_html = unicode_html.encode("iso-8859-1") - - # Parse the ISO-Latin-1 HTML. - soup = self.soup(iso_latin_html) - # Encode it to UTF-8. - result = soup.encode("utf-8") - - # What do we expect the result to look like? Well, it would - # look like unicode_html, except that the META tag would say - # UTF-8 instead of ISO-Latin-1. - expected = unicode_html.replace("ISO-Latin-1", "utf-8") - - # And, of course, it would be in UTF-8, not Unicode. - expected = expected.encode("utf-8") - - # Ta-da! - self.assertEqual(result, expected) - - def test_real_shift_jis_document(self): - # Smoke test to make sure the parser can handle a document in - # Shift-JIS encoding, without choking. - shift_jis_html = ( - b'<html><head></head><body><pre>' - b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f' - b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c' - b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B' - b'</pre></body></html>') - unicode_html = shift_jis_html.decode("shift-jis") - soup = self.soup(unicode_html) - - # Make sure the parse tree is correctly encoded to various - # encodings. - self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8")) - self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp")) - - # Tests below this line need work. - - def test_meta_tag_reflects_current_encoding(self): - # Here's the <meta> tag saying that a document is - # encoded in Shift-JIS. - meta_tag = ('<meta content="text/html; charset=x-sjis" ' - 'http-equiv="Content-type"/>') - - # Here's a document incorporating that meta tag. - shift_jis_html = ( - '<html><head>\n%s\n' - '<meta http-equiv="Content-language" content="ja"/>' - '</head><body>Shift-JIS markup goes here.') % meta_tag - soup = self.soup(shift_jis_html) - - # Parse the document, and the charset is replaced with a - # generic value. - parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'}) - self.assertEqual(parsed_meta['content'], - 'text/html; charset=%SOUP-ENCODING%') - self.assertEqual(parsed_meta.contains_substitutions, True) - - # For the rest of the story, see TestSubstitutions in - # test_tree.py. - - def test_entities_converted_on_the_way_out(self): - text = "<p><<sacré bleu!>></p>" - expected = u"<<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>".encode("utf-8") - soup = self.soup(text) - str = soup.p.string - #self.assertEqual(str.encode("utf-8"), expected) - - def test_br_tag_is_empty_element(self): - """A <br> tag is designated as an empty-element tag.""" - soup = self.soup("<br></br>") - self.assertTrue(soup.br.is_empty_element) - self.assertEqual(str(soup.br), "<br/>") - - def test_p_tag_is_not_empty_element(self): - """A <p> tag is not designated as an empty-element tag.""" - soup = self.soup("<p/>") - self.assertFalse(soup.p.is_empty_element) - self.assertEqual(str(soup.p), "<p></p>") - - def test_soupstrainer(self): - strainer = SoupStrainer("b") - soup = self.soup("A <b>bold</b> <meta/> <i>statement</i>", - parse_only=strainer) - self.assertEqual(soup.decode(), "<b>bold</b>") - - def test_deepcopy(self): - # Make sure you can copy the builder. This is important because - # the builder is part of a BeautifulSoup object, and we want to be - # able to copy that. - copy.deepcopy(self.default_builder) - -class TestHTMLParserTreeBuilderInvalidMarkup(SoupTest): - """Tests of invalid markup for the default tree builder. - - Subclass this to test other builders. - - These are very likely to give different results for different tree - builders. It's not required that a tree builder handle invalid - markup at all. - """ - - @property - def default_builder(self): - return HTMLParserTreeBuilder() - - def test_table_containing_bare_markup(self): - # Markup should be in table cells, not directly in the table. - self.assertSoupEquals("<table><div>Foo</div></table>") - - def test_incorrectly_nested_table(self): - # The second <table> tag is floating in the <tr> tag - # rather than being inside a <td>. - bad_markup = ('<table id="1">' - '<tr>' - "<td>Here's another table:</td>" - '<table id="2">' - '<tr><td>foo</td></tr>' - '</table></td>') - - - def test_unclosed_a_tag(self): - # <a> tags really ought to be closed at some point. - # - # We have all the <div> tags because HTML5 says to duplicate - # the <a> tag rather than closing it, and that's what html5lib - # does. - markup = """<div id="1"> - <a href="foo"> -</div> -<div id="2"> - <div id="3"> - <a href="bar"></a> - </div> -</div>""" - - expect = """<div id="1"> -<a href="foo"> -</a></div> -<div id="2"> -<div id="3"> -<a href="bar"></a> -</div> -</div>""" - self.assertSoupEquals(markup, expect) - - def test_unclosed_block_level_elements(self): - # Unclosed block-level elements should be closed. - self.assertSoupEquals( - '<blockquote><p><b>Foo</blockquote><p>Bar', - '<blockquote><p><b>Foo</b></p></blockquote><p>Bar</p>') - - def test_fake_self_closing_tag(self): - # If a self-closing tag presents as a normal tag, it's treated - # as one. - self.assertSoupEquals( - "<item><link>http://foo.com/</link></item>", - "<item><link>http://foo.com/</link></item>") - - def test_boolean_attribute_with_no_value(self): - soup = self.soup("<table><td nowrap>foo</td></table>") - self.assertEqual(soup.table.td['nowrap'], None) - - def test_incorrectly_nested_tables(self): - self.assertSoupEquals( - '<table><tr><table><tr id="nested">', - '<table><tr><table><tr id="nested"></tr></table></tr></table>') - - def test_floating_text_in_table(self): - self.assertSoupEquals("<table><td></td>foo<td>bar</td></table>") - - def test_paragraphs_containing_block_display_elements(self): - markup = self.soup("<p>this is the definition:" - "<dl><dt>first case</dt>") - # The <p> tag is not closed before the <dl> tag begins. - self.assertEqual(len(markup.p.contents), 2) - - def test_empty_element_tag_with_contents(self): - self.assertSoupEquals("<br>foo</br>", "<br>foo</br>") - - def test_doctype_in_body(self): - markup = "<p>one<!DOCTYPE foobar>two</p>" - self.assertSoupEquals(markup) - - def test_nonsensical_declaration(self): - # Declarations that don't make any sense are ignored. - self.assertRaises(HTMLParseError, self.soup, '<! Foo = -8><p>a</p>') - - def test_whitespace_in_doctype(self): - # A declaration that has extra whitespace is ignored. - self.assertRaises( - HTMLParseError, self.soup, - '<! DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">' + - '<p>foo</p>') - - def test_incomplete_declaration(self): - self.assertRaises(HTMLParseError, self.soup, 'a<!b <p>c') - - def test_cdata_where_it_doesnt_belong(self): - #CDATA sections are ignored. - markup = "<div><![CDATA[foo]]>" - soup = self.soup(markup) - self.assertEquals(soup.div.contents[0], CData("foo")) - - def test_attribute_value_never_got_closed(self): - markup = '<a href="http://foo.com/</a> and blah and blah' - soup = self.soup(markup) - self.assertEqual(soup.encode(), b"") - - def test_attribute_value_with_embedded_brackets(self): - soup = self.soup('<a b="<a>">') - self.assertEqual(soup.a['b'], '<a>') - - def test_nonexistent_entity(self): - soup = self.soup("<p>foo&#bar;baz</p>") - # This is very strange. - self.assertEqual(soup.p.string, "foo<p") - - # Compare a real entity. - soup = self.soup("<p>foodbaz</p>") - self.assertEqual(soup.p.string, "foodbaz") - - # Also compare html5lib, which preserves the &# before the - # entity name. - - def test_entity_out_of_range(self): - # An entity that's out of range will be replaced with - # REPLACEMENT CHARACTER. - soup = self.soup("<p>�</p>") - self.assertEqual(soup.p.string, u"\N{REPLACEMENT CHARACTER}") - - soup = self.soup("<p>�</p>") - self.assertEqual(soup.p.string, u"\N{REPLACEMENT CHARACTER}") - - soup = self.soup("<p>�</p>") - self.assertEqual(soup.p.string, u"\N{REPLACEMENT CHARACTER}") - - - def test_entity_was_not_finished(self): - soup = self.soup("<p><Hello>") - # Compare html5lib, which completes the entity. - self.assertEqual(soup.p.string, "<Hello") - - def test_document_ends_with_incomplete_declaration(self): - soup = self.soup('<p>a<!b') - # This becomes a string 'a'. The incomplete declaration is ignored. - # Compare html5lib, which turns it into a comment. - self.assertEqual(soup.p.contents, ['a']) - - def test_document_starts_with_bogus_declaration(self): - self.assertRaises(HTMLParseError, self.soup, '<! Foo ><p>a</p>') - - def test_tag_name_contains_unicode(self): - # Unicode characters in tag names are stripped. - tag_name = u"<our\N{SNOWMAN}>Joe</our\N{SNOWMAN}>" - self.assertSoupEquals("<our>Joe</our>") - - def test_multiple_values_for_the_same_attribute(self): - markup = '<b b="20" a="1" b="10" a="2" a="3" a="4"></b>' - self.assertSoupEquals(markup, '<b a="4" b="10"></b>') - -class TestHTMLParserTreeBuilderEncodingConversion(SoupTest): - # Test Beautiful Soup's ability to decode and encode from various - # encodings. - - @property - def default_builder(self): - return HTMLParserTreeBuilder() - - def setUp(self): - super(TestHTMLParserTreeBuilderEncodingConversion, self).setUp() - self.unicode_data = u"<html><head></head><body><foo>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</foo></body></html>" - self.utf8_data = self.unicode_data.encode("utf-8") - # Just so you know what it looks like. - self.assertEqual( - self.utf8_data, - b"<html><head></head><body><foo>Sacr\xc3\xa9 bleu!</foo></body></html>") - - def test_ascii_in_unicode_out(self): - # ASCII input is converted to Unicode. The original_encoding - # attribute is set. - ascii = b"<foo>a</foo>" - soup_from_ascii = self.soup(ascii) - unicode_output = soup_from_ascii.decode() - self.assertTrue(isinstance(unicode_output, unicode)) - self.assertEqual(unicode_output, self.document_for(ascii.decode())) - self.assertEqual(soup_from_ascii.original_encoding, "ascii") - - def test_unicode_in_unicode_out(self): - # Unicode input is left alone. The original_encoding attribute - # is not set. - soup_from_unicode = self.soup(self.unicode_data) - self.assertEqual(soup_from_unicode.decode(), self.unicode_data) - self.assertEqual(soup_from_unicode.foo.string, u'Sacr\xe9 bleu!') - self.assertEqual(soup_from_unicode.original_encoding, None) - - def test_utf8_in_unicode_out(self): - # UTF-8 input is converted to Unicode. The original_encoding - # attribute is set. - soup_from_utf8 = self.soup(self.utf8_data) - self.assertEqual(soup_from_utf8.decode(), self.unicode_data) - self.assertEqual(soup_from_utf8.foo.string, u'Sacr\xe9 bleu!') - - def test_utf8_out(self): - # The internal data structures can be encoded as UTF-8. - soup_from_unicode = self.soup(self.unicode_data) - self.assertEqual(soup_from_unicode.encode('utf-8'), self.utf8_data) - - HEBREW_DOCUMENT = b'<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>' - - def test_real_hebrew_document(self): - # A real-world test to make sure we can convert ISO-8859-9 (a - # Hebrew encoding) to UTF-8. - soup = self.soup(self.HEBREW_DOCUMENT, - from_encoding="iso-8859-8") - self.assertEqual(soup.original_encoding, 'iso-8859-8') - self.assertEqual( - soup.encode('utf-8'), - self.HEBREW_DOCUMENT.decode("iso-8859-8").encode("utf-8")) + # html.parser can't handle namespaced doctypes, so skip this one. + pass diff --git a/bs4/tests/test_lxml.py b/bs4/tests/test_lxml.py index 4d19e7f..92b7389 100644 --- a/bs4/tests/test_lxml.py +++ b/bs4/tests/test_lxml.py @@ -12,238 +12,26 @@ from bs4 import BeautifulSoup from bs4.element import Comment, Doctype, SoupStrainer from bs4.testing import skipIf from bs4.tests import test_htmlparser -from bs4.testing import skipIf +from bs4.testing import ( + HTMLTreeBuilderSmokeTest, + SoupTest, + skipIf, +) @skipIf( not LXML_PRESENT, "lxml seems not to be present, not testing its tree builder.") -class TestLXMLTreeBuilder(test_htmlparser.TestHTMLParserTreeBuilder): - """A smoke test for the LXML tree builder. - - Subclass this to test some other HTML tree builder. Subclasses of - this test ensure that all of Beautiful Soup's tree builders - generate more or less the same trees. - - It's okay for trees to differ--just override the appropriate test - method to demonstrate how one tree builder differs from the LXML - builder. But in general, all HTML tree builders should generate - trees that make most of these tests pass. - """ +class LXMLTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest): + """See ``HTMLTreeBuilderSmokeTest``.""" @property def default_builder(self): return LXMLTreeBuilder() - def test_bare_string(self): - # A bare string is turned into some kind of HTML document or - # fragment recognizable as the original string. - # - # In this case, lxml puts a <p> tag around the bare string. - self.assertSoupEquals( - "A bare string", "<p>A bare string</p>") - - def test_cdata_where_its_ok(self): - # lxml strips CDATA sections, no matter where they occur. - markup = "<svg><![CDATA[foobar]]>" - self.assertSoupEquals(markup, "<svg></svg>") - - def test_empty_element(self): - # HTML's empty-element tags are recognized as such. - self.assertSoupEquals( - "<p>A <meta> tag</p>", "<p>A <meta/> tag</p>") - + def test_out_of_range_entity(self): self.assertSoupEquals( - "<p>Foo<br/>bar</p>", "<p>Foo<br/>bar</p>") - - def test_naked_ampersands(self): - # Ampersands are left alone. - text = "<p>AT&T</p>" - soup = self.soup(text) - self.assertEqual(soup.p.string, "AT&T") - - # Even if they're in attribute values. - invalid_url = '<a href="http://example.org?a=1&b=2;3">foo</a>' - soup = self.soup(invalid_url) - self.assertEqual(soup.a['href'], "http://example.org?a=1&b=2;3") - - def test_literal_in_textarea(self): - # Anything inside a <textarea> is supposed to be treated as - # the literal value of the field, (XXX citation - # needed). html5lib does this correctly. But, lxml does its - # best to parse the contents of a <textarea> as HTML. - text = '<textarea>Junk like <b> tags and <&<&</textarea>' - soup = self.soup(text) - self.assertEqual(len(soup.textarea.contents), 2) - self.assertEqual(soup.textarea.contents[0], u"Junk like ") - self.assertEqual(soup.textarea.contents[1].name, 'b') - self.assertEqual(soup.textarea.b.string, u" tags and ") - - def test_literal_in_script(self): - # The contents of a <script> tag are treated as a literal string, - # even if that string contains HTML. - javascript = 'if (i < 2) { alert("<b>foo</b>"); }' - soup = self.soup('<script>%s</script>' % javascript) - self.assertEqual(soup.script.string, javascript) - - def test_doctype(self): - # Test a normal HTML doctype you'll commonly see in a real document. - self._test_doctype( - 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"') - - def test_namespaced_system_doctype(self): - # Test a namespaced doctype with a system id. - self._test_doctype('xsl:stylesheet SYSTEM "htmlent.dtd"') - - def test_namespaced_public_doctype(self): - # Test a namespaced doctype with a public id. - self._test_doctype('xsl:stylesheet PUBLIC "htmlent.dtd"') - - def test_entities_in_attribute_values_converted_during_parsing(self): - - # The numeric entity isn't recognized without the closing - # semicolon. - text = '<x t="piñata">' - expected = u"pi\N{LATIN SMALL LETTER N WITH TILDE}ata" - soup = self.soup(text) - self.assertEqual(soup.x['t'], expected) - - text = '<x t="piñata">' - expected = u"pi\N{LATIN SMALL LETTER N WITH TILDE}ata" - soup = self.soup(text) - self.assertEqual(soup.x['t'], u"pi\xf1ata") - - text = '<x t="piñata">' - soup = self.soup(text) - self.assertEqual(soup.x['t'], expected) - - text = '<x t="sacré bleu">' - soup = self.soup(text) - self.assertEqual( - soup.x['t'], - u"sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu") - - # This can cause valid HTML to become invalid. - valid_url = '<a href="http://example.org?a=1&b=2;3">foo</a>' - soup = self.soup(valid_url) - self.assertEqual(soup.a['href'], "http://example.org?a=1&b=2;3") - - -@skipIf( - not LXML_PRESENT, - "lxml seems not to be present, not testing it on invalid markup.") -class TestLXMLTreeBuilderInvalidMarkup( - test_htmlparser.TestHTMLParserTreeBuilderInvalidMarkup): - - @property - def default_builder(self): - return LXMLTreeBuilder() - - def test_attribute_value_never_got_closed(self): - markup = '<a href="http://foo.com/</a> and blah and blah' - soup = self.soup(markup) - self.assertEqual( - soup.a['href'], "http://foo.com/</a> and blah and blah") - - def test_attribute_value_was_closed_by_subsequent_tag(self): - markup = """<a href="foo</a>, </a><a href="bar">baz</a>""" - soup = self.soup(markup) - # The string between the first and second quotes was interpreted - # as the value of the 'href' attribute. - self.assertEqual(soup.a['href'], 'foo</a>, </a><a href=') - - #The string after the second quote (bar"), was treated as an - #empty attribute called bar. - self.assertEqual(soup.a['bar'], '') - self.assertEqual(soup.a.string, "baz") - - def test_document_starts_with_bogus_declaration(self): - soup = self.soup('<! Foo ><p>a</p>') - # The declaration is ignored altogether. - self.assertEqual(soup.encode(), b"<html><body><p>a</p></body></html>") - - def test_incomplete_declaration(self): - # An incomplete declaration will screw up the rest of the document. - self.assertSoupEquals('a<!b <p>c', '<p>a</p>') - - def test_nonsensical_declaration(self): - # Declarations that don't make any sense are ignored. - self.assertSoupEquals('<! Foo = -8><p>a</p>', "<p>a</p>") - - def test_unquoted_attribute_value(self): - soup = self.soup('<a style={height:21px;}></a>') - self.assertEqual(soup.a['style'], '{height:21px;}') - - def test_whitespace_in_doctype(self): - # A declaration that has extra whitespace is ignored. + "<p>foo�bar</p>", "<p>foobar</p>") self.assertSoupEquals( - ('<! DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">' - '<p>foo</p>'), - '<p>foo</p>') - - def test_boolean_attribute_with_no_value(self): - soup = self.soup("<table><td nowrap>foo</td></table>") - self.assertEqual(soup.table.td['nowrap'], '') - - def test_cdata_where_it_doesnt_belong(self): - #CDATA sections are ignored. - markup = "<div><![CDATA[foo]]>" - self.assertSoupEquals(markup, "<div></div>") - - def test_empty_element_tag_with_contents(self): - self.assertSoupEquals("<br>foo</br>", "<br/>foo") - - def test_nonexistent_entity(self): - soup = self.soup("<p>foo&#bar;baz</p>") - self.assertEqual(soup.p.string, "foobar;baz") - - # Compare a real entity. - soup = self.soup("<p>foodbaz</p>") - self.assertEqual(soup.p.string, "foodbaz") - - # Also compare html5lib, which preserves the &# before the - # entity name. - - def test_entity_was_not_finished(self): - soup = self.soup("<p><Hello>") - # Compare html5lib, which completes the entity. - self.assertEqual(soup.p.string, "<Hello>") - - def test_fake_self_closing_tag(self): - # If a self-closing tag presents as a normal tag, the 'open' - # tag is treated as an instance of the self-closing tag and - # the 'close' tag is ignored. + "<p>foo�bar</p>", "<p>foobar</p>") self.assertSoupEquals( - "<item><link>http://foo.com/</link></item>", - "<item><link/>http://foo.com/</item>") - - def test_paragraphs_containing_block_display_elements(self): - markup = self.soup("<p>this is the definition:" - "<dl><dt>first case</dt>") - # The <p> tag is closed before the <dl> tag begins. - self.assertEqual(markup.p.contents, ["this is the definition:"]) - - def test_multiple_values_for_the_same_attribute(self): - markup = '<b b="20" a="1" b="10" a="2" a="3" a="4"></b>' - self.assertSoupEquals(markup, '<b a="1" b="20"></b>') - - def test_entity_out_of_range(self): - # An entity that's out of range will be ignored. - soup = self.soup("<p>�</p>") - self.assertEqual(0, len(soup.p.contents)) - - soup = self.soup("<p>�</p>") - self.assertEqual(0, len(soup.p.contents)) - - soup = self.soup("<p>�</p>") - self.assertEqual(0, len(soup.p.contents)) - - -@skipIf( - not LXML_PRESENT, - "lxml seems not to be present, not testing it on encoding conversion.") -class TestLXMLParserTreeBuilderEncodingConversion( - test_htmlparser.TestHTMLParserTreeBuilderEncodingConversion): - - @property - def default_builder(self): - return LXMLTreeBuilder() + "<p>foo�bar</p>", "<p>foobar</p>") diff --git a/bs4/tests/test_soup.py b/bs4/tests/test_soup.py index 896e914..2b7c003 100644 --- a/bs4/tests/test_soup.py +++ b/bs4/tests/test_soup.py @@ -103,6 +103,51 @@ class TestEntitySubstitution(unittest.TestCase): text = 'Bob\'s "bar"' self.assertEqual(self.sub.substitute_html(text), text) + +class TestEncodingConversion(SoupTest): + # Test Beautiful Soup's ability to decode and encode from various + # encodings. + + def setUp(self): + super(TestEncodingConversion, self).setUp() + self.unicode_data = u"<html><head></head><body><foo>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</foo></body></html>" + self.utf8_data = self.unicode_data.encode("utf-8") + # Just so you know what it looks like. + self.assertEqual( + self.utf8_data, + b"<html><head></head><body><foo>Sacr\xc3\xa9 bleu!</foo></body></html>") + + def test_ascii_in_unicode_out(self): + # ASCII input is converted to Unicode. The original_encoding + # attribute is set. + ascii = b"<foo>a</foo>" + soup_from_ascii = self.soup(ascii) + unicode_output = soup_from_ascii.decode() + self.assertTrue(isinstance(unicode_output, unicode)) + self.assertEqual(unicode_output, self.document_for(ascii.decode())) + self.assertEqual(soup_from_ascii.original_encoding, "ascii") + + def test_unicode_in_unicode_out(self): + # Unicode input is left alone. The original_encoding attribute + # is not set. + soup_from_unicode = self.soup(self.unicode_data) + self.assertEqual(soup_from_unicode.decode(), self.unicode_data) + self.assertEqual(soup_from_unicode.foo.string, u'Sacr\xe9 bleu!') + self.assertEqual(soup_from_unicode.original_encoding, None) + + def test_utf8_in_unicode_out(self): + # UTF-8 input is converted to Unicode. The original_encoding + # attribute is set. + soup_from_utf8 = self.soup(self.utf8_data) + self.assertEqual(soup_from_utf8.decode(), self.unicode_data) + self.assertEqual(soup_from_utf8.foo.string, u'Sacr\xe9 bleu!') + + def test_utf8_out(self): + # The internal data structures can be encoded as UTF-8. + soup_from_unicode = self.soup(self.unicode_data) + self.assertEqual(soup_from_unicode.encode('utf-8'), self.utf8_data) + + class TestUnicodeDammit(unittest.TestCase): """Standalone tests of Unicode, Dammit.""" diff --git a/scripts/demonstrate_parser_differences.py b/scripts/demonstrate_parser_differences.py new file mode 100644 index 0000000..d84670a --- /dev/null +++ b/scripts/demonstrate_parser_differences.py @@ -0,0 +1,95 @@ +"""Demonstrate how different parsers parse the same markup. + +Beautiful Soup can use any of a number of different parsers. Every +parser should behave more or less the same on valid markup, and +Beautiful Soup's unit tests make sure this is the case. But every +parser handles invalid markup differently. Even different versions of +the same parser handle invalid markup differently. So instead of unit +tests I've created this educational demonstration script. + +The file demonstration_markup.txt contains many lines of HTML. This +script tests each line of markup against every parser you have +installed, and prints out how each parser sees that markup. This may +help you choose a parser, or understand why Beautiful Soup presents +your document the way it does. +""" + +import os +import sys +from bs4 import BeautifulSoup +parsers = ['html.parser'] + +try: + from bs4.builder import _lxml + parsers.append('lxml') +except ImportError, e: + pass + +try: + from bs4.builder import _html5lib + parsers.append('html5lib') +except ImportError, e: + pass + +class Demonstration(object): + def __init__(self, markup): + self.results = {} + self.markup = markup + + def run_against(self, *parser_names): + uniform_results = True + previous_output = None + for parser in parser_names: + try: + soup = BeautifulSoup(self.markup, parser) + if markup.startswith("<div>"): + # Extract the interesting part + output = soup.div + else: + output = soup + except Exception, e: + output = "[EXCEPTION] %s" % str(e) + self.results[parser] = output + if previous_output is None: + previous_output = output + elif previous_output != output: + uniform_results = False + return uniform_results + + def dump(self): + print "%s: %s" % ("Markup".rjust(13), self.markup.encode("utf8")) + for parser, output in self.results.items(): + print "%s: %s" % (parser.rjust(13), output.encode("utf8")) + +different_results = [] +uniform_results = [] + +print "= Testing the following parsers: %s =" % ", ".join(parsers) +print + +input_file = sys.stdin +if sys.stdin.isatty(): + for filename in [ + "demonstration_markup.txt", + os.path.join("scripts", "demonstration_markup.txt")]: + if os.path.exists(filename): + input_file = open(filename) + +for markup in input_file: + demo = Demonstration(markup.decode("utf8").strip().replace("\\n", "\n")) + is_uniform = demo.run_against(*parsers) + if is_uniform: + uniform_results.append(demo) + else: + different_results.append(demo) + +print "== Markup that's handled the same in every parser ==" +print +for demo in uniform_results: + demo.dump() + print +print "== Markup that's not handled the same in every parser ==" +print +for demo in different_results: + demo.dump() + print diff --git a/scripts/demonstration_markup.txt b/scripts/demonstration_markup.txt new file mode 100644 index 0000000..a7914a0 --- /dev/null +++ b/scripts/demonstration_markup.txt @@ -0,0 +1,34 @@ +A bare string +<!DOCTYPE xsl:stylesheet SYSTEM "htmlent.dtd"> +<!DOCTYPE xsl:stylesheet PUBLIC "htmlent.dtd"> +<div><![CDATA[A CDATA section where it doesn't belong]]></div> +<div><svg><![CDATA[HTML5 does allow CDATA sections in SVG]]></svg></div> +<div>A <meta> tag</div> +<div>A <br> tag that supposedly has contents.</br></div> +<div>AT&T</div> +<div><textarea>Within a textarea, markup like <b> tags and <&<& should be treated as literal</textarea></div> +<div><script>if (i < 2) { alert("<b>Markup within script tags should be treated as literal.</b>"); }</script></div> +<div>This numeric entity is missing the final semicolon: <x t="piñata"></div> +<div><a href="http://example.com/</a> that attribute value never got closed</div> +<div><a href="foo</a>, </a><a href="bar">that attribute value was closed by the subsequent tag</a></div> +<! This document starts with a bogus declaration ><div>a</div> +<div>This document contains <!an incomplete declaration <div>(do you see it?)</div> +<div>This document ends with <!an incomplete declaration +<div><a style={height:21px;}>That attribute value was bogus</a></div> +<! DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">The doctype is invalid because it contains extra whitespace +<div><table><td nowrap>That boolean attribute had no value</td></table></div> +<div>Here's a nonexistent entity: &#foo; (do you see it?)</div> +<div>This document ends before the entity finishes: > +<div><p>Paragraphs shouldn't contain block display elements, but this one does: <dl><dt>you see?</dt></p> +<b b="20" a="1" b="10" a="2" a="3" a="4">Multiple values for the same attribute.</b> +<div><table><tr><td>Here's a table</td></tr></table></div> +<div><table id="1"><tr><td>Here's a nested table:<table id="2"><tr><td>foo</td></tr></table></td></div> +<div>This tag contains nothing but whitespace: <b> </b></div> +<div><blockquote><p><b>This p tag is cut off by</blockquote></p>the end of the blockquote tag</div> +<div><table><div>This table contains bare markup</div></table></div> +<div><div id="1">\n <a href="link1">This link is never closed.\n</div>\n<div id="2">\n <div id="3">\n <a href="link2">This link is closed.</a>\n </div>\n</div></div> +<div>This document contains a <!DOCTYPE surprise>surprise doctype</div> +<div><a><B><Cd><EFG>Mixed case tags are folded to lowercase</efg></CD></b></A></div> +<div><our☃>Tag name contains Unicode characters</our☃></div> +<div><a ☃="snowman">Attribute name contains Unicode characters</a></div> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |