1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
|
"""Tests to ensure that the lxml tree builder generates good trees."""
from beautifulsoup import BeautifulSoup
from beautifulsoup.builder.lxml_builder import LXMLTreeBuilder
from beautifulsoup.element import Comment
from beautifulsoup.testing import SoupTest
class TestLXMLBuilder(SoupTest):
"""A smoke test for the LXML tree builders.
Subclass this to test some other tree builder. Subclasses of this
test ensure that all of Beautiful Soup's tree builders generate
more or less the same trees. It's okay for trees to differ,
especially when given invalid markup--just override the
appropriate test method to demonstrate how one tree builder
differs from the LXML builder.
"""
def test_bare_string(self):
# A bare string is turned into some kind of HTML document or
# fragment recognizable as the original string.
#
# In this case, lxml puts a <p> tag around the bare string.
self.assertSoupEquals(
"A bare string", "<p>A bare string</p>")
def test_mixed_case_tags(self):
# Mixed-case tags are folded to lowercase.
self.assertSoupEquals(
"<a><B><Cd><EFG></efg></CD></b></A>",
"<a><b><cd><efg></efg></cd></b></a>")
def test_self_closing(self):
# HTML's self-closing tags are recognized as such.
self.assertSoupEquals(
"<p>A <meta> tag</p>", "<p>A <meta /> tag</p>")
self.assertSoupEquals(
"<p>Foo<br/>bar</p>", "<p>Foo<br />bar</p>")
def test_comment(self):
# Comments are represented as Comment objects.
markup = "<p>foo<!--foobar-->baz</p>"
self.assertSoupEquals(markup)
soup = self.soup(markup)
comment = soup.find(text="foobar")
self.assertEquals(comment.__class__, Comment)
def test_nested_inline_elements(self):
# Inline tags can be nested indefinitely.
b_tag = "<b>Inside a B tag</b>"
self.assertSoupEquals(b_tag)
nested_b_tag = "<p>A <i>nested <b>tag</b></i></p>"
self.assertSoupEquals(nested_b_tag)
double_nested_b_tag = "<p>A <a>doubly <i>nested <b>tag</b></i></a></p>"
self.assertSoupEquals(nested_b_tag)
def test_nested_block_level_elements(self):
soup = self.soup('<blockquote><p><b>Foo</b></p></blockquote>')
blockquote = soup.blockquote
self.assertEqual(blockquote.p.b.string, 'Foo')
self.assertEqual(blockquote.b.string, 'Foo')
def test_collapsed_whitespace(self):
"""In most tags, whitespace is collapsed."""
self.assertSoupEquals("<p> </p>", "<p> </p>")
def test_preserved_whitespace_in_pre_and_textarea(self):
"""In <pre> and <textarea> tags, whitespace is preserved."""
self.assertSoupEquals("<pre> </pre>")
self.assertSoupEquals("<textarea> woo </textarea>")
def test_single_quote_attribute_values_become_double_quotes(self):
self.assertSoupEquals("<foo attr='bar'></foo>",
'<foo attr="bar"></foo>')
def test_attribute_values_with_nested_quotes_are_left_alone(self):
text = """<foo attr='bar "brawls" happen'>a</foo>"""
self.assertSoupEquals(text)
def test_attribute_values_with_double_nested_quotes_get_quoted(self):
text = """<foo attr='bar "brawls" happen'>a</foo>"""
soup = self.soup(text)
soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
self.assertSoupEquals(
soup.foo.decode(),
"""<foo attr='Brawls happen at "Bob&squot;s Bar"'>a</foo>""")
def test_ampersand_in_attribute_value_gets_quoted(self):
self.assertSoupEquals('<this is="really messed up & stuff"></this>',
'<this is="really messed up & stuff"></this>')
def test_literal_in_textarea(self):
# Anything inside a <textarea> is supposed to be treated as
# the literal value of the field, (XXX citation needed).
#
# But, both lxml and html5lib do their best to parse the
# contents of a <textarea> as HTML.
text = '<textarea>Junk like <b> tags and <&<&</textarea>'
soup = BeautifulSoup(text)
self.assertEquals(len(soup.textarea.contents), 2)
self.assertEquals(soup.textarea.contents[0], u"Junk like ")
self.assertEquals(soup.textarea.contents[1].name, 'b')
self.assertEquals(soup.textarea.b.string, u" tags and ")
def test_literal_in_script(self):
# The contents of a <script> tag are treated as a literal string,
# even if that string contains HTML.
javascript = 'if (i < 2) { alert("<b>foo</b>"); }'
soup = BeautifulSoup('<script>%s</script>' % javascript)
self.assertEquals(soup.script.string, javascript)
def test_naked_ampersands(self):
# Ampersands are left alone.
text = "<p>AT&T</p>"
soup = self.soup(text)
self.assertEquals(soup.p.string, "AT&T")
# Even if they're in attribute values.
invalid_url = '<a href="http://example.org?a=1&b=2;3">foo</a>'
soup = self.soup(invalid_url)
self.assertEquals(soup.a['href'], "http://example.org?a=1&b=2;3")
def test_entities_in_strings_converted_during_parsing(self):
# Both XML and HTML entities are converted to Unicode characters
# during parsing.
text = "<p><<sacré bleu!>></p>"
expected = u"<p><<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></p>"
self.assertSoupEquals(text, expected)
def test_entities_in_attribute_values_converted_during_parsing(self):
text = '<x t="piñata">'
expected = u"pi\N{LATIN SMALL LETTER N WITH TILDE}ata"
soup = self.soup(text)
self.assertEquals(soup.x['t'], expected)
text = '<x t="piñata">'
soup = self.soup(text)
self.assertEquals(soup.x['t'], expected)
text = '<x t="sacré bleu">'
soup = self.soup(text)
self.assertEquals(
soup.x['t'],
u"sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu")
# This can cause valid HTML to become invalid.
valid_url = '<a href="http://example.org?a=1&b=2;3">foo</a>'
soup = self.soup(valid_url)
self.assertEquals(soup.a['href'], "http://example.org?a=1&b=2;3")
def test_smart_quotes_converted_on_the_way_in(self):
# Microsoft smart quotes are converted to Unicode characters during
# parsing.
quote = "<p>\x91Foo\x92</p>"
soup = self.soup(quote)
self.assertEquals(
soup.p.string,
u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
def test_non_breaking_spaces_converted_on_the_way_in(self):
soup = self.soup("<a> </a>")
self.assertEquals(soup.a.string, u"\N{NO-BREAK SPACE}" * 2)
# Tests below this line need work.
def test_entities_converted_on_the_way_out(self):
text = "<p><<sacré bleu!>></p>"
expected = u"<<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>".encode("utf-8")
soup = BeautifulSoup(text)
str = soup.p.string
#self.assertEquals(str.encode("utf-8"), expected)
def test_foo(self):
isolatin = """<html><meta http-equiv="Content-type" content="text/html; charset=ISO-Latin-1" />Sacr\xe9 bleu!</html>"""
soup = self.soup(isolatin)
utf8 = isolatin.replace("ISO-Latin-1".encode(), "utf-8".encode())
utf8 = utf8.replace("\xe9", "\xc3\xa9")
#print soup
class TestLXMLBuilderInvalidMarkup(SoupTest):
"""Tests of invalid markup for the LXML tree builder.
Subclass this to test other builders.
These are very likely to give different results for different tree
builders. It's not required that a tree builder handle invalid
markup at all.
"""
def test_unclosed_block_level_elements(self):
# Unclosed block-level elements should be closed.
self.assertSoupEquals(
'<blockquote><p><b>Foo</blockquote><p>Bar',
'<blockquote><p><b>Foo</b></p></blockquote><p>Bar</p>')
def test_fake_self_closing_tag(self):
# If a self-closing tag presents as a normal tag, the 'open'
# tag is treated as an instance of the self-closing tag and
# the 'close' tag is ignored.
self.assertSoupEquals(
"<item><link>http://foo.com/</link></item>",
"<item><link />http://foo.com/</item>")
def test_boolean_attribute_with_no_value_gets_empty_value(self):
soup = self.soup("<table><td nowrap>foo</td></table>")
self.assertEquals(soup.table.td['nowrap'], '')
def test_incorrectly_nested_tables(self):
self.assertSoupEquals(
'<table><tr><table><tr id="nested">',
'<table><tr><table><tr id="nested"></tr></table></tr></table>')
|
