try: from bs4.builder import HTML5TreeBuilder HTML5LIB_PRESENT = True except ImportError, e: HTML5LIB_PRESENT = False from bs4.element import Comment, SoupStrainer import test_htmlparser import unittest from bs4.testing import skipIf @skipIf( not HTML5LIB_PRESENT, "html5lib seems not to be present, not testing its tree builder.") class TestHTML5Builder(test_htmlparser.TestHTMLParserTreeBuilder): """See `BuilderSmokeTest`.""" @property def default_builder(self): return HTML5TreeBuilder() def test_soupstrainer(self): # The html5lib tree builder does not support SoupStrainers. strainer = SoupStrainer("b") markup = "

A bold statement.

" soup = self.soup(markup, parse_only=strainer) self.assertEqual( soup.decode(), self.document_for(markup)) def test_bare_string(self): # A bare string is turned into some kind of HTML document or # fragment recognizable as the original string. # # In this case, html5lib puts a

tag around the bare string. self.assertSoupEquals( "A bare string", "A bare string") def test_correctly_nested_tables(self): markup = ('' '' "') self.assertSoupEquals( markup, '
Here's another table:" '' '' '
foo
Here\'s another table:' '
foo
' '
') self.assertSoupEquals( "" "" "
Foo
Bar
Baz
") def test_literal_in_textarea(self): markup = '' soup = self.soup(markup) self.assertEqual( soup.textarea.contents, ["Junk like tags and <&<&"]) def test_collapsed_whitespace(self): """Whitespace is preserved even in tags that don't require it.""" self.assertSoupEquals("

") self.assertSoupEquals(" ") def test_cdata_where_its_ok(self): # In html5lib 0.9.0, all CDATA sections are converted into # comments. In a later version (unreleased as of this # writing), CDATA sections in tags like and will # be preserved. BUT, I'm not sure how Beautiful Soup needs to # adjust to transform this preservation into the construction # of a BS CData object. markup = "foobar" # Eventually we should be able to do a find(text="foobar") and # get a CData object. self.assertSoupEquals(markup, "") def test_entities_in_attribute_values_converted_during_parsing(self): # The numeric entity is recognized even without the closing # semicolon. text = '' expected = u"pi\N{LATIN SMALL LETTER N WITH TILDE}ata" soup = self.soup(text) self.assertEqual(soup.x['t'], expected) def test_naked_ampersands(self): # Ampersands are not treated as entities, unlike in html.parser. text = "

AT&T

" soup = self.soup(text) self.assertEqual(soup.p.string, "AT&T") def test_namespaced_system_doctype(self): # Test a namespaced doctype with a system id. self._test_doctype('xsl:stylesheet SYSTEM "htmlent.dtd"') def test_namespaced_public_doctype(self): # Test a namespaced doctype with a public id. self._test_doctype('xsl:stylesheet PUBLIC "htmlent.dtd"') @skipIf( not HTML5LIB_PRESENT, "html5lib seems not to be present, not testing it on invalid markup.") class TestHTML5BuilderInvalidMarkup( test_htmlparser.TestHTMLParserTreeBuilderInvalidMarkup): """See `BuilderInvalidMarkupSmokeTest`.""" @property def default_builder(self): return HTML5TreeBuilder() def test_unclosed_block_level_elements(self): # The unclosed tag is closed so that the block-level tag # can be closed, and another tag is inserted after the # next block-level tag begins. self.assertSoupEquals( '

Foo

Bar', '

Foo

Bar

') def test_attribute_value_never_got_closed(self): markup = ' and blah and blah") def test_attribute_value_was_closed_by_subsequent_tag(self): markup = """baz""" soup = self.soup(markup) # The string between the first and second quotes was interpreted # as the value of the 'href' attribute. self.assertEqual(soup.a['href'], 'foo,

a

') # The declaration is ignored altogether. self.assertEqual(soup.encode(), b"

a

") def test_table_containing_bare_markup(self): # Markup should be in table cells, not directly in the table. self.assertSoupEquals("
Foo
", "
Foo
") def test_unclosed_a_tag(self): # n.b. the whitespace is important here. markup = """ """ expect = """ """ self.assertSoupEquals(markup, expect) def test_incorrectly_nested_tables(self): self.assertSoupEquals( '
', ('
' '
')) def test_floating_text_in_table(self): self.assertSoupEquals( "foo
bar
", "foo
bar
") def test_empty_element_tag_with_contents(self): self.assertSoupEquals("
foo
", "
foo
") def test_doctype_in_body(self): markup = "

onetwo

" self.assertSoupEquals(markup, "

onetwo

") def test_cdata_where_it_doesnt_belong(self): # Random CDATA sections are converted into comments. markup = "
" soup = self.soup(markup) data = soup.find(text="[CDATA[foo]]") self.assertEqual(data.__class__, Comment) def test_nonsensical_declaration(self): # Declarations that don't make any sense are turned into comments. soup = self.soup('

a

') self.assertEqual(str(soup), ("" "

a

")) soup = self.soup('

a

') self.assertEqual(str(soup), ("

a

" "")) def test_whitespace_in_doctype(self): # A declaration that has extra whitespace is turned into a comment. soup = self.soup(( '' '

foo

')) self.assertEqual( str(soup), ('' '

foo

')) def test_incomplete_declaration(self): # An incomplete declaration is treated as a comment. markup = 'ac' self.assertSoupEquals(markup, "ac") # Let's spell that out a little more explicitly. soup = self.soup(markup) str1, comment, str2 = soup.body.contents self.assertEqual(str1, 'a') self.assertEqual(comment.__class__, Comment) self.assertEqual(comment, 'b a') # 'Foo' becomes a comment that appears before the HTML. comment = soup.contents[0] self.assertTrue(isinstance(comment, Comment)) self.assertEqual(comment, 'Foo') self.assertEqual(self.find(text="a") == "a") def test_attribute_value_was_closed_by_subsequent_tag(self): markup = """baz""" soup = self.soup(markup) # The string between the first and second quotes was interpreted # as the value of the 'href' attribute. self.assertEqual(soup.a['href'], 'foo,

a

') # The declaration becomes a comment. comment = soup.contents[0] self.assertTrue(isinstance(comment, Comment)) self.assertEqual(comment, ' Foo ') self.assertEqual(soup.p.string, 'a') def test_document_ends_with_incomplete_declaration(self): soup = self.soup('

a<Hello>") # Compare html5lib, which completes the entity. self.assertEqual(soup.p.string, "") def test_nonexistent_entity(self): soup = self.soup("

foo&#bar;baz

") self.assertEqual(soup.p.string, "foo&#bar;baz") # Compare a real entity. soup = self.soup("

foodbaz

") self.assertEqual(soup.p.string, "foodbaz") def test_entity_out_of_range(self): # An entity that's out of range will be converted to # REPLACEMENT CHARACTER. soup = self.soup("

�

") self.assertEqual(soup.p.string, u"\N{REPLACEMENT CHARACTER}") soup = self.soup("

�

") self.assertEqual(soup.p.string, u"\N{REPLACEMENT CHARACTER}") def test_incomplete_declaration(self): self.assertSoupEquals('ac', 'ac') def test_nonsensical_declaration(self): soup = self.soup('

a

') self.assertEquals( soup.decode(), "

a

") def test_unquoted_attribute_value(self): soup = self.soup('') self.assertEqual(soup.a['style'], '{height:21px;}') def test_boolean_attribute_with_no_value(self): soup = self.soup("
foo
") self.assertEqual(soup.table.td['nowrap'], '') def test_cdata_where_it_doesnt_belong(self): #CDATA sections are ignored. markup = "
" self.assertSoupEquals(markup, "
") def test_empty_element_tag_with_contents(self): self.assertSoupEquals("
foo
", "
foo
") def test_fake_self_closing_tag(self): # If a self-closing tag presents as a normal tag, the 'open' # tag is treated as an instance of the self-closing tag and # the 'close' tag is ignored. self.assertSoupEquals( "http://foo.com/", "http://foo.com/") def test_paragraphs_containing_block_display_elements(self): markup = self.soup("

this is the definition:" "

first case
") # The

tag is closed before the

tag begins. self.assertEqual(markup.p.contents, ["this is the definition:"]) def test_multiple_values_for_the_same_attribute(self): markup = '' self.assertSoupEquals(markup, '') @skipIf( not HTML5LIB_PRESENT, "html5lib seems not to be present, not testing it on encoding conversion.") class TestHTML5LibEncodingConversion( test_htmlparser.TestHTMLParserTreeBuilderEncodingConversion): @property def default_builder(self): return HTML5TreeBuilder() def test_real_hebrew_document(self): # A real-world test to make sure we can convert ISO-8859-8 (a # Hebrew encoding) to UTF-8. soup = self.soup(self.HEBREW_DOCUMENT, from_encoding="iso-8859-8") self.assertEqual(soup.original_encoding, 'iso8859-8') self.assertEqual( soup.encode('utf-8'), self.HEBREW_DOCUMENT.decode("iso-8859-8").encode("utf-8"))