Ported some more tests, fixed an encoding problem, and added rudimentary doctype handling.

1 files changed, 117 insertions, 2 deletions

diff --git a/tests/test_lxml.py b/tests/test_lxml.py
index d16e8d9..9a65f6a 100644
--- a/tests/test_lxml.py
+++ b/tests/test_lxml.py
@@ -1,5 +1,7 @@
 """Tests to ensure that the lxml tree builder generates good trees."""
 
+import re
+
 from beautifulsoup import BeautifulSoup
 from beautifulsoup.builder.lxml_builder import LXMLTreeBuilder
 from beautifulsoup.element import Comment
@@ -65,6 +67,34 @@ class TestLXMLBuilder(SoupTest):
         self.assertEqual(blockquote.p.b.string, 'Foo')
         self.assertEqual(blockquote.b.string, 'Foo')
 
+    # This is a <table> tag containing another <table> tag in one of its
+    # cells.
+    TABLE_MARKUP_1 = ('<table id="1">'
+                     '<tr>'
+                     "<td>Here's another table:"
+                     '<table id="2">'
+                     '<tr><td>foo</td></tr>'
+                     '</table></td>')
+
+    def test_correctly_nested_tables(self):
+        markup = ('<table id="1">'
+                  '<tr>'
+                  "<td>Here's another table:"
+                  '<table id="2">'
+                  '<tr><td>foo</td></tr>'
+                  '</table></td>')
+
+        self.assertSoupEquals(
+            markup,
+            '<table id="1"><tr><td>Here\'s another table:'
+            '<table id="2"><tr><td>foo</td></tr></table>'
+            '</td></tr></table>')
+
+        self.assertSoupEquals(
+            "<table><thead><tr><td>Foo</td></tr></thead>"
+            "<tbody><tr><td>Bar</td></tr></tbody>"
+            "<tfoot><tr><td>Baz</td></tr></tfoot></table>")
+
     def test_collapsed_whitespace(self):
         """In most tags, whitespace is collapsed."""
         self.assertSoupEquals("<p>   </p>", "<p> </p>")
@@ -114,14 +144,82 @@ class TestLXMLBuilder(SoupTest):
         soup = BeautifulSoup('<script>%s</script>' % javascript)
         self.assertEquals(soup.script.string, javascript)
 
+    def test_naked_ampersands(self):
+        # Ampersands are left alone.
+        text = "<p>AT&T</p>"
+        soup = self.soup(text)
+        self.assertEquals(soup.p.string, "AT&T")
+
+        # Even if they're in attribute values.
+        invalid_url = '<a href="http://example.org?a=1&b=2;3">foo</a>'
+        soup = self.soup(invalid_url)
+        self.assertEquals(soup.a['href'], "http://example.org?a=1&b=2;3")
+
+    def test_entities_in_strings_converted_during_parsing(self):
+        # Both XML and HTML entities are converted to Unicode characters
+        # during parsing.
+        text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
+        expected = u"<p><<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></p>"
+        self.assertSoupEquals(text, expected)
+
+    def test_entities_in_attribute_values_converted_during_parsing(self):
+        text = '<x t="pi&#241ata">'
+        expected = u"pi\N{LATIN SMALL LETTER N WITH TILDE}ata"
+        soup = self.soup(text)
+        self.assertEquals(soup.x['t'], expected)
+
+        text = '<x t="pi&#xf1;ata">'
+        soup = self.soup(text)
+        self.assertEquals(soup.x['t'], expected)
+
+        text = '<x t="sacr&eacute; bleu">'
+        soup = self.soup(text)
+        self.assertEquals(
+            soup.x['t'],
+            u"sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu")
+
+        # This can cause valid HTML to become invalid.
+        valid_url = '<a href="http://example.org?a=1&amp;b=2;3">foo</a>'
+        soup = self.soup(valid_url)
+        self.assertEquals(soup.a['href'], "http://example.org?a=1&b=2;3")
+
+    def test_smart_quotes_converted_on_the_way_in(self):
+        # Microsoft smart quotes are converted to Unicode characters during
+        # parsing.
+        quote = "<p>\x91Foo\x92</p>"
+        soup = self.soup(quote)
+        self.assertEquals(
+            soup.p.string,
+            u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
+
+    def test_non_breaking_spaces_converted_on_the_way_in(self):
+        soup = self.soup("<a>&nbsp;&nbsp;</a>")
+        self.assertEquals(soup.a.string, u"\N{NO-BREAK SPACE}" * 2)
+
+    # Tests below this line need work.
+
+    #def test_doctype(self):
+    #    xml = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"><html>foo</html></p>'
+    #    self.assertSoupEquals(xml)
+
+
+    #def test_cdata(self):
+    #    print self.soup("<div><![CDATA[foo]]></div>")
+
+    def test_entities_converted_on_the_way_out(self):
+        text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
+        expected = u"&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;".encode("utf-8")
+        soup = BeautifulSoup(text)
+        str = soup.p.string
+        #self.assertEquals(str.encode("utf-8"), expected)
+
     def test_foo(self):
         isolatin = """<html><meta http-equiv="Content-type" content="text/html; charset=ISO-Latin-1" />Sacr\xe9 bleu!</html>"""
         soup = self.soup(isolatin)
 
         utf8 = isolatin.replace("ISO-Latin-1".encode(), "utf-8".encode())
         utf8 = utf8.replace("\xe9", "\xc3\xa9")
-
-        print soup
+        #print soup
 
 
 class TestLXMLBuilderInvalidMarkup(SoupTest):
@@ -134,6 +232,20 @@ class TestLXMLBuilderInvalidMarkup(SoupTest):
     markup at all.
     """
 
+    def test_table_containing_bare_markup(self):
+        # Markup should be in table cells, not directly in the table.
+        self.assertSoupEquals("<table><div>Foo</div></table>")
+
+    def test_incorrectly_nested_table(self):
+        # The second <table> tag is floating in the <tr> tag
+        # rather than being inside a <td>.
+        bad_markup = ('<table id="1">'
+                      '<tr>'
+                      "<td>Here's another table:</td>"
+                      '<table id="2">'
+                      '<tr><td>foo</td></tr>'
+                      '</table></td>')
+
     def test_unclosed_block_level_elements(self):
         # Unclosed block-level elements should be closed.
         self.assertSoupEquals(
@@ -157,4 +269,7 @@ class TestLXMLBuilderInvalidMarkup(SoupTest):
             '<table><tr><table><tr id="nested">',
             '<table><tr><table><tr id="nested"></tr></table></tr></table>')
 
+    def test_doctype_in_body(self):
+        markup = "<p>one<!DOCTYPE foobar>two</p>"
+        self.assertSoupEquals(markup)