1 files changed, 51 insertions, 0 deletions
diff --git a/tests/test_lxml.py b/tests/test_lxml.py
index b002227..0f54e78 100644
--- a/tests/test_lxml.py
+++ b/tests/test_lxml.py
@@ -375,3 +375,54 @@ class TestLXMLBuilderInvalidMarkup(SoupTest):
         #CDATA sections are ignored.
         markup = "<div><![CDATA[foo]]>"
         self.assertSoupEquals(markup, "<div></div>")
+
+
+class TestLXMLBuilderEncodingConversion(SoupTest):
+    # Test Beautiful Soup's ability to decode and encode from various
+    # encodings.
+
+    def setUp(self):
+        super(TestLXMLBuilderEncodingConversion, self).setUp()
+        self.unicode_data = u"<html><body><foo>\xe9</foo></body></html>"
+        self.utf8_data = self.unicode_data.encode("utf-8")
+        self.assertEqual(
+            self.utf8_data, "<html><body><foo>\xc3\xa9</foo></body></html>")
+
+    def test_ascii_in_unicode_out(self):
+        # ASCII input is converted to Unicode. The originalEncoding
+        # attribute is set.
+        ascii = "<foo>a</foo>"
+        soup_from_ascii = self.soup(ascii)
+        unicode_output = soup_from_ascii.decode()
+        self.assertTrue(isinstance(unicode_output, unicode))
+        self.assertEquals(unicode_output, self.document_for(ascii))
+        self.assertEquals(soup_from_ascii.originalEncoding, "ascii")
+
+    def test_unicode_in_unicode_out(self):
+        # Unicode input is left alone. The originalEncoding attribute
+        # is not set.
+        soup_from_unicode = self.soup(self.unicode_data)
+        self.assertEquals(soup_from_unicode.decode(), self.unicode_data)
+        self.assertEquals(soup_from_unicode.foo.string, u'\xe9')
+        self.assertEquals(soup_from_unicode.originalEncoding, None)
+
+    def test_utf8_in_unicode_out(self):
+        # UTF-8 input is converted to Unicode. The originalEncoding
+        # attribute is set.
+        soup_from_utf8 = self.soup(self.utf8_data)
+        self.assertEquals(soup_from_utf8.decode(), self.unicode_data)
+        self.assertEquals(soup_from_utf8.foo.string, u'\xe9')
+
+    def test_utf8_out(self):
+        # The internal data structures can be encoded as UTF-8.
+        soup_from_unicode = self.soup(self.unicode_data)
+        self.assertEquals(soup_from_unicode.encode('utf-8'), self.utf8_data)
+
+    def test_real_hebrew_document(self):
+        # A real-world test to make sure we can convert ISO-8859-9 (a
+        # Hebrew encoding) to UTF-8.
+        iso_8859_8= '<HTML><HEAD><TITLE>Hebrew (ISO 8859-8) in Visual Directionality</TITLE></HEAD><BODY><H1>Hebrew (ISO 8859-8) in Visual Directionality</H1>\xed\xe5\xec\xf9</BODY></HTML>'
+        utf8 = '<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xd7\x9d\xd7\x95\xd7\x9c\xd7\xa9</body></html>'
+        soup = self.soup(iso_8859_8, fromEncoding="iso-8859-8")
+        self.assertEquals(soup.originalEncoding, 'iso-8859-8')
+        self.assertEquals(soup.encode('utf-8'), utf8)