1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
|
from beautifulsoup.builder.html5lib_builder import HTML5TreeBuilder
from beautifulsoup.element import Comment
from test_lxml import (
TestLXMLBuilder,
TestLXMLBuilderInvalidMarkup,
TestLXMLBuilderEncodingConversion,
)
class TestHTML5Builder(TestLXMLBuilder):
"""See `BuilderSmokeTest`."""
@property
def default_builder(self):
return HTML5TreeBuilder()
def test_bare_string(self):
# A bare string is turned into some kind of HTML document or
# fragment recognizable as the original string.
#
# In this case, lxml puts a <p> tag around the bare string.
self.assertSoupEquals(
"A bare string", "A bare string")
def test_correctly_nested_tables(self):
markup = ('<table id="1">'
'<tr>'
"<td>Here's another table:"
'<table id="2">'
'<tr><td>foo</td></tr>'
'</table></td>')
self.assertSoupEquals(
markup,
'<table id="1"><tbody><tr><td>Here\'s another table:'
'<table id="2"><tbody><tr><td>foo</td></tr></tbody></table>'
'</td></tr></tbody></table>')
self.assertSoupEquals(
"<table><thead><tr><td>Foo</td></tr></thead>"
"<tbody><tr><td>Bar</td></tr></tbody>"
"<tfoot><tr><td>Baz</td></tr></tfoot></table>")
def test_literal_in_textarea(self):
markup = '<textarea>Junk like <b> tags and <&<&</textarea>'
soup = self.soup(markup)
self.assertEquals(
soup.textarea.contents, ["Junk like <b> tags and <&<&"])
def test_collapsed_whitespace(self):
"""Whitespace is preserved even in tags that don't require it."""
self.assertSoupEquals("<p> </p>")
self.assertSoupEquals("<b> </b>")
def test_cdata_where_its_ok(self):
# In html5lib 0.9.0, all CDATA sections are converted into
# comments. In a later version (unreleased as of this
# writing), CDATA sections in tags like <svg> and <math> will
# be preserved. BUT, I'm not sure how Beautiful Soup needs to
# adjust to transform this preservation into the construction
# of a BS CData object.
markup = "<svg><![CDATA[foobar]]>"
# Eventually we should be able to do a find(text="foobar") and
# get a CData object.
self.assertSoupEquals(markup, "<svg><!--[CDATA[foobar]]--></svg>")
class TestHTML5BuilderInvalidMarkup(TestLXMLBuilderInvalidMarkup):
"""See `BuilderInvalidMarkupSmokeTest`."""
@property
def default_builder(self):
return HTML5TreeBuilder()
def test_unclosed_block_level_elements(self):
# The unclosed <b> tag is closed so that the block-level tag
# can be closed, and another <b> tag is inserted after the
# next block-level tag begins.
self.assertSoupEquals(
'<blockquote><p><b>Foo</blockquote><p>Bar',
'<blockquote><p><b>Foo</b></p></blockquote><p><b>Bar</b></p>')
def test_table_containing_bare_markup(self):
# Markup should be in table cells, not directly in the table.
self.assertSoupEquals("<table><div>Foo</div></table>",
"<div>Foo</div><table></table>")
def test_incorrectly_nested_tables(self):
self.assertSoupEquals(
'<table><tr><table><tr id="nested">',
('<table><tbody><tr></tr></tbody></table>'
'<table><tbody><tr id="nested"></tr></tbody></table>'))
def test_doctype_in_body(self):
markup = "<p>one<!DOCTYPE foobar>two</p>"
self.assertSoupEquals(markup, "<p>onetwo</p>")
def test_cdata_where_it_doesnt_belong(self):
# Random CDATA sections are converted into comments.
markup = "<div><![CDATA[foo]]>"
soup = self.soup(markup)
data = soup.find(text="[CDATA[foo]]")
self.assertEquals(data.__class__, Comment)
def test_nonsensical_declaration(self):
# Declarations that don't make any sense are turned into comments.
soup = self.soup('<! Foo = -8><p>a</p>')
self.assertEquals(str(soup),
("<!-- Foo = -8-->"
"<html><head></head><body><p>a</p></body></html>"))
soup = self.soup('<p>a</p><! Foo = -8>')
self.assertEquals(str(soup),
("<html><head></head><body><p>a</p>"
"<!-- Foo = -8--></body></html>"))
def test_whitespace_in_doctype(self):
# A declaration that has extra whitespace is turned into a comment.
soup = self.soup((
'<! DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">'
'<p>foo</p>'))
self.assertEquals(
str(soup),
('<!-- DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"-->'
'<html><head></head><body><p>foo</p></body></html>'))
def test_incomplete_declaration(self):
# An incomplete declaration is treated as a comment.
markup = 'a<!b <p>c'
self.assertSoupEquals(markup, "a<!--b <p-->c")
# Let's spell that out a little more explicitly.
soup = self.soup(markup)
str1, comment, str2 = soup.body.contents
self.assertEquals(str1, 'a')
self.assertEquals(comment.__class__, Comment)
self.assertEquals(comment, 'b <p')
self.assertEquals(str2, 'c')
def test_document_starts_with_bogus_declaration(self):
soup = self.soup('<! Foo >a')
# 'Foo' becomes a comment that appears before the HTML.
comment = soup.contents[0]
self.assertTrue(isinstance(comment, Comment))
self.assertEquals(comment, 'Foo')
self.assertEquals(self.find(text="a") == "a")
def test_attribute_value_was_closed_by_subsequent_tag(self):
markup = """<a href="foo</a>, </a><a href="bar">baz</a>"""
soup = self.soup(markup)
# The string between the first and second quotes was interpreted
# as the value of the 'href' attribute.
self.assertEquals(soup.a['href'], 'foo</a>, </a><a href=')
#The string after the second quote (bar"), was treated as an
#empty attribute called bar".
self.assertEquals(soup.a['bar"'], '')
self.assertEquals(soup.a.string, "baz")
def test_document_starts_with_bogus_declaration(self):
soup = self.soup('<! Foo ><p>a</p>')
# The declaration becomes a comment.
comment = soup.contents[0]
self.assertTrue(isinstance(comment, Comment))
self.assertEquals(comment, ' Foo ')
self.assertEquals(soup.p.string, 'a')
def test_document_ends_with_incomplete_declaration(self):
soup = self.soup('<p>a<!b')
# This becomes a string 'a'. The incomplete declaration is ignored.
# Compare html5lib, which turns it into a comment.
s, comment = soup.p.contents
self.assertEquals(s, 'a')
self.assertTrue(isinstance(comment, Comment))
self.assertEquals(comment, 'b')
def test_entity_was_not_finished(self):
soup = self.soup("<p><Hello>")
# Compare html5lib, which completes the entity.
self.assertEquals(soup.p.string, "<Hello>")
def test_nonexistent_entity(self):
soup = self.soup("<p>foo&#bar;baz</p>")
self.assertEquals(soup.p.string, "foo&#bar;baz")
# Compare a real entity.
soup = self.soup("<p>foodbaz</p>")
self.assertEquals(soup.p.string, "foodbaz")
class TestHTML5LibEncodingConversion(TestLXMLBuilderEncodingConversion):
@property
def default_builder(self):
return HTML5TreeBuilder()
def test_real_hebrew_document(self):
# A real-world test to make sure we can convert ISO-8859-9 (a
# Hebrew encoding) to UTF-8.
soup = self.soup(self.HEBREW_DOCUMENT,
fromEncoding="iso-8859-8")
self.assertEquals(soup.original_encoding, 'iso8859-8')
self.assertEquals(
soup.encode('utf-8'),
self.HEBREW_DOCUMENT.decode("iso-8859-8").encode("utf-8"))
|
