diff options
| author | Leonard Richardson <leonard.richardson@canonical.com> | 2011-02-18 12:10:10 -0500 |
|---|---|---|
| committer | Leonard Richardson <leonard.richardson@canonical.com> | 2011-02-18 12:10:10 -0500 |
| commit | 0dda99b15112df7225e647db9702fbd62dcc8ea8 (patch) | |
| tree | 1127d44d52716738835c6ab2128fdb1561bc7cc2 /tests/test_lxml.py | |
| parent | 66cbef12d959149746b3361f227f2a0328a31469 (diff) | |
| parent | 945b719a28c229178e710b749d2af4d00a81bdba (diff) | |
Defer to html5lib's Unicode converter rather than using Unicode, Dammit. The lxml treebuilder still uses UD.
Diffstat (limited to 'tests/test_lxml.py')
| -rw-r--r-- | tests/test_lxml.py | 56 |
1 files changed, 56 insertions, 0 deletions
diff --git a/tests/test_lxml.py b/tests/test_lxml.py index b002227..4c11b1d 100644 --- a/tests/test_lxml.py +++ b/tests/test_lxml.py @@ -375,3 +375,59 @@ class TestLXMLBuilderInvalidMarkup(SoupTest): #CDATA sections are ignored. markup = "<div><![CDATA[foo]]>" self.assertSoupEquals(markup, "<div></div>") + + +class TestLXMLBuilderEncodingConversion(SoupTest): + # Test Beautiful Soup's ability to decode and encode from various + # encodings. + + def setUp(self): + super(TestLXMLBuilderEncodingConversion, self).setUp() + self.unicode_data = u"<html><head></head><body><foo>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</foo></body></html>" + self.utf8_data = self.unicode_data.encode("utf-8") + # Just so you know what it looks like. + self.assertEqual( + self.utf8_data, + "<html><head></head><body><foo>Sacr\xc3\xa9 bleu!</foo></body></html>") + + def test_ascii_in_unicode_out(self): + # ASCII input is converted to Unicode. The originalEncoding + # attribute is set. + ascii = "<foo>a</foo>" + soup_from_ascii = self.soup(ascii) + unicode_output = soup_from_ascii.decode() + self.assertTrue(isinstance(unicode_output, unicode)) + self.assertEquals(unicode_output, self.document_for(ascii)) + self.assertEquals(soup_from_ascii.originalEncoding, "ascii") + + def test_unicode_in_unicode_out(self): + # Unicode input is left alone. The originalEncoding attribute + # is not set. + soup_from_unicode = self.soup(self.unicode_data) + self.assertEquals(soup_from_unicode.decode(), self.unicode_data) + self.assertEquals(soup_from_unicode.foo.string, u'Sacr\xe9 bleu!') + self.assertEquals(soup_from_unicode.originalEncoding, None) + + def test_utf8_in_unicode_out(self): + # UTF-8 input is converted to Unicode. The originalEncoding + # attribute is set. + soup_from_utf8 = self.soup(self.utf8_data) + self.assertEquals(soup_from_utf8.decode(), self.unicode_data) + self.assertEquals(soup_from_utf8.foo.string, u'Sacr\xe9 bleu!') + + def test_utf8_out(self): + # The internal data structures can be encoded as UTF-8. + soup_from_unicode = self.soup(self.unicode_data) + self.assertEquals(soup_from_unicode.encode('utf-8'), self.utf8_data) + + HEBREW_DOCUMENT = '<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>' + + def test_real_hebrew_document(self): + # A real-world test to make sure we can convert ISO-8859-9 (a + # Hebrew encoding) to UTF-8. + soup = self.soup(self.HEBREW_DOCUMENT, + fromEncoding="iso-8859-8") + self.assertEquals(soup.originalEncoding, 'iso-8859-8') + self.assertEquals( + soup.encode('utf-8'), + self.HEBREW_DOCUMENT.decode("iso-8859-8").encode("utf-8")) |