Defer to html5lib's Unicode converter rather than using Unicode, Dammit. The lxml treebuilder still uses UD.

8 files changed, 182 insertions, 25 deletions

diff --git a/beautifulsoup/__init__.py b/beautifulsoup/__init__.py
index f2c20de..32ea73f 100644
--- a/beautifulsoup/__init__.py
+++ b/beautifulsoup/__init__.py
@@ -144,37 +144,29 @@ class BeautifulStoneSoup(Tag):
         self.builder.soup = self
 
         self.parseOnlyThese = parseOnlyThese
-        self.fromEncoding = fromEncoding
 
         self.reset()
 
         if hasattr(markup, 'read'):        # It's a file-type object.
             markup = markup.read()
-        self.markup = markup
+        self.markup, self.originalEncoding, self.declaredHTMLEncoding = (
+            self.builder.prepare_markup(markup, fromEncoding))
+
         try:
-            self._feed(isHTML=self.builder.assume_html)
+            self._feed()
         except StopParsing:
             pass
-        self.markup = None                 # The markup can now be GCed.
+
+        # Clear out the markup and the builder so they can be CGed.
+        self.markup = None
         self.builder.soup = None
-        self.builder = None                # So can the builder.
+        self.builder = None
 
-    def _feed(self, inDocumentEncoding=None, isHTML=False):
+    def _feed(self):
         # Convert the document to Unicode.
-        markup = self.markup
-        if isinstance(markup, unicode):
-            if not hasattr(self, 'originalEncoding'):
-                self.originalEncoding = None
-        else:
-            dammit = UnicodeDammit\
-                     (markup, [self.fromEncoding, inDocumentEncoding],
-                      isHTML=isHTML)
-            markup = dammit.unicode
-            self.originalEncoding = dammit.originalEncoding
-            self.declaredHTMLEncoding = dammit.declaredHTMLEncoding
         self.builder.reset()
 
-        self.builder.feed(markup)
+        self.builder.feed(self.markup)
         # Close out any unfinished strings and close all the open tags.
         self.endData()
         while self.currentTag.name != self.ROOT_TAG_NAME:
diff --git a/beautifulsoup/builder/__init__.py b/beautifulsoup/builder/__init__.py
index cf5e6c6..5bf5929 100644
--- a/beautifulsoup/builder/__init__.py
+++ b/beautifulsoup/builder/__init__.py
@@ -25,6 +25,10 @@ class TreeBuilder(Entities):
     def feed(self, markup):
         raise NotImplementedError()
 
+    def prepare_markup(self, markup, user_specified_encoding=None,
+                       document_declared_encoding=None):
+        return markup, None, None
+
     def test_fragment_to_document(self, fragment):
         """Wrap an HTML fragment to make it look like a document.
 
diff --git a/beautifulsoup/builder/html5lib_builder.py b/beautifulsoup/builder/html5lib_builder.py
index dc95493..95151da 100644
--- a/beautifulsoup/builder/html5lib_builder.py
+++ b/beautifulsoup/builder/html5lib_builder.py
@@ -13,10 +13,23 @@ from beautifulsoup.element import (
 class HTML5TreeBuilder(HTMLTreeBuilder):
     """Use html5lib to build a tree."""
 
+    def prepare_markup(self, markup, user_specified_encoding):
+        # Store the user-specified encoding for use later on.
+        self.user_specified_encoding = user_specified_encoding
+        return markup, None, None
+
     # These methods are defined by Beautiful Soup.
     def feed(self, markup):
         parser = html5lib.HTMLParser(tree=self.create_treebuilder)
-        doc = parser.parse(markup)
+        doc = parser.parse(markup, encoding=self.user_specified_encoding)
+
+        # Set the character encoding detected by the tokenizer.
+        if isinstance(markup, unicode):
+            # We need to special-case this because html5lib sets
+            # charEncoding to UTF-8 if it gets Unicode input.
+            doc.originalEncoding = None
+        else:
+            doc.originalEncoding = parser.tokenizer.stream.charEncoding[0]
 
     def create_treebuilder(self, namespaceHTMLElements):
         self.underlying_builder = TreeBuilderForHtml5lib(
diff --git a/beautifulsoup/builder/lxml_builder.py b/beautifulsoup/builder/lxml_builder.py
index 9ced9f0..a1f8c1e 100644
--- a/beautifulsoup/builder/lxml_builder.py
+++ b/beautifulsoup/builder/lxml_builder.py
@@ -1,6 +1,7 @@
 from lxml import etree
 from beautifulsoup.element import Comment, Doctype
 from beautifulsoup.builder import HTMLTreeBuilder
+from beautifulsoup.dammit import UnicodeDammit
 
 class LXMLTreeBuilder(HTMLTreeBuilder):
 
@@ -11,6 +12,20 @@ class LXMLTreeBuilder(HTMLTreeBuilder):
         self.parser = parser_class(target=self)
         self.soup = None
 
+    def prepare_markup(self, markup, user_specified_encoding=None,
+                       document_declared_encoding=None):
+        """
+        :return: A 3-tuple (markup, original encoding, encoding
+        declared within markup).
+        """
+        if isinstance(markup, unicode):
+            return markup, None, None
+
+        try_encodings = [user_specified_encoding, document_declared_encoding]
+        dammit = UnicodeDammit(markup, try_encodings, isHTML=True)
+        return dammit.markup, dammit.originalEncoding, dammit.declaredHTMLEncoding
+
+
     def feed(self, markup):
         self.parser.feed(markup)
         self.parser.close()
diff --git a/beautifulsoup/dammit.py b/beautifulsoup/dammit.py
index 78bd4b2..954ca54 100644
--- a/beautifulsoup/dammit.py
+++ b/beautifulsoup/dammit.py
@@ -58,13 +58,12 @@ class UnicodeDammit:
             return
 
         u = None
-        for proposedEncoding in overrideEncodings:
-            u = self._convertFrom(proposedEncoding)
-            if u: break
-        if not u:
-            for proposedEncoding in (documentEncoding, sniffedEncoding):
+        for proposedEncoding in (
+            overrideEncodings + [documentEncoding, sniffedEncoding]):
+            if proposedEncoding is not None:
                 u = self._convertFrom(proposedEncoding)
-                if u: break
+                if u:
+                    break
 
         # If no luck and we have auto-detection library, try that:
         if not u and chardet and not isinstance(self.markup, unicode):
diff --git a/tests/test_html5lib.py b/tests/test_html5lib.py
index 3efdebf..59d84a3 100644
--- a/tests/test_html5lib.py
+++ b/tests/test_html5lib.py
@@ -3,6 +3,7 @@ from beautifulsoup.element import Comment
 from test_lxml import (
     TestLXMLBuilder,
     TestLXMLBuilderInvalidMarkup,
+    TestLXMLBuilderEncodingConversion,
     )
 
 class TestHTML5Builder(TestLXMLBuilder):
@@ -138,3 +139,19 @@ class TestHTML5BuilderInvalidMarkup(TestLXMLBuilderInvalidMarkup):
         utf8 = utf8.replace("\xe9", "\xc3\xa9")
 
         #print soup
+
+
+class TestHTML5LibEncodingConversion(TestLXMLBuilderEncodingConversion):
+    @property
+    def default_builder(self):
+        return HTML5TreeBuilder()
+
+    def test_real_hebrew_document(self):
+        # A real-world test to make sure we can convert ISO-8859-9 (a
+        # Hebrew encoding) to UTF-8.
+        soup = self.soup(self.HEBREW_DOCUMENT,
+                         fromEncoding="iso-8859-8")
+        self.assertEquals(soup.originalEncoding, 'iso8859-8')
+        self.assertEquals(
+            soup.encode('utf-8'),
+            self.HEBREW_DOCUMENT.decode("iso-8859-8").encode("utf-8"))
diff --git a/tests/test_lxml.py b/tests/test_lxml.py
index b002227..4c11b1d 100644
--- a/tests/test_lxml.py
+++ b/tests/test_lxml.py
@@ -375,3 +375,59 @@ class TestLXMLBuilderInvalidMarkup(SoupTest):
         #CDATA sections are ignored.
         markup = "<div><![CDATA[foo]]>"
         self.assertSoupEquals(markup, "<div></div>")
+
+
+class TestLXMLBuilderEncodingConversion(SoupTest):
+    # Test Beautiful Soup's ability to decode and encode from various
+    # encodings.
+
+    def setUp(self):
+        super(TestLXMLBuilderEncodingConversion, self).setUp()
+        self.unicode_data = u"<html><head></head><body><foo>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</foo></body></html>"
+        self.utf8_data = self.unicode_data.encode("utf-8")
+        # Just so you know what it looks like.
+        self.assertEqual(
+            self.utf8_data,
+            "<html><head></head><body><foo>Sacr\xc3\xa9 bleu!</foo></body></html>")
+
+    def test_ascii_in_unicode_out(self):
+        # ASCII input is converted to Unicode. The originalEncoding
+        # attribute is set.
+        ascii = "<foo>a</foo>"
+        soup_from_ascii = self.soup(ascii)
+        unicode_output = soup_from_ascii.decode()
+        self.assertTrue(isinstance(unicode_output, unicode))
+        self.assertEquals(unicode_output, self.document_for(ascii))
+        self.assertEquals(soup_from_ascii.originalEncoding, "ascii")
+
+    def test_unicode_in_unicode_out(self):
+        # Unicode input is left alone. The originalEncoding attribute
+        # is not set.
+        soup_from_unicode = self.soup(self.unicode_data)
+        self.assertEquals(soup_from_unicode.decode(), self.unicode_data)
+        self.assertEquals(soup_from_unicode.foo.string, u'Sacr\xe9 bleu!')
+        self.assertEquals(soup_from_unicode.originalEncoding, None)
+
+    def test_utf8_in_unicode_out(self):
+        # UTF-8 input is converted to Unicode. The originalEncoding
+        # attribute is set.
+        soup_from_utf8 = self.soup(self.utf8_data)
+        self.assertEquals(soup_from_utf8.decode(), self.unicode_data)
+        self.assertEquals(soup_from_utf8.foo.string, u'Sacr\xe9 bleu!')
+
+    def test_utf8_out(self):
+        # The internal data structures can be encoded as UTF-8.
+        soup_from_unicode = self.soup(self.unicode_data)
+        self.assertEquals(soup_from_unicode.encode('utf-8'), self.utf8_data)
+
+    HEBREW_DOCUMENT = '<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>'
+
+    def test_real_hebrew_document(self):
+        # A real-world test to make sure we can convert ISO-8859-9 (a
+        # Hebrew encoding) to UTF-8.
+        soup = self.soup(self.HEBREW_DOCUMENT,
+                         fromEncoding="iso-8859-8")
+        self.assertEquals(soup.originalEncoding, 'iso-8859-8')
+        self.assertEquals(
+            soup.encode('utf-8'),
+            self.HEBREW_DOCUMENT.decode("iso-8859-8").encode("utf-8"))
diff --git a/tests/test_soup.py b/tests/test_soup.py
new file mode 100644
index 0000000..4fb2142
--- /dev/null
+++ b/tests/test_soup.py
@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+"""Tests of Beautiful Soup as a whole."""
+
+import unittest
+from beautifulsoup.element import SoupStrainer
+from beautifulsoup.dammit import UnicodeDammit
+from beautifulsoup.testing import SoupTest
+
+
+class TestSelectiveParsing(SoupTest):
+
+    def test_parse_with_soupstrainer(self):
+        markup = "No<b>Yes</b><a>No<b>Yes <c>Yes</c></b>"
+        strainer = SoupStrainer("b")
+        soup = self.soup(markup, parseOnlyThese=strainer)
+        self.assertEquals(soup.encode(), "<b>Yes</b><b>Yes <c>Yes</c></b>")
+
+
+class TestUnicodeDammit(unittest.TestCase):
+    """Standalone tests of Unicode, Dammit."""
+
+    def test_smart_quotes_to_xml_entities(self):
+        markup = "<foo>\x91\x92\x93\x94</foo>"
+        dammit = UnicodeDammit(markup)
+        self.assertEquals(
+            dammit.unicode, "<foo>&#x2018;&#x2019;&#x201C;&#x201D;</foo>")
+
+    def test_smart_quotes_to_html_entities(self):
+        markup = "<foo>\x91\x92\x93\x94</foo>"
+        dammit = UnicodeDammit(markup, smartQuotesTo="html")
+        self.assertEquals(
+            dammit.unicode, "<foo>&lsquo;&rsquo;&ldquo;&rdquo;</foo>")
+
+    def test_detect_utf8(self):
+        utf8 = "\xc3\xa9"
+        dammit = UnicodeDammit(utf8)
+        self.assertEquals(dammit.unicode, u'\xe9')
+        self.assertEquals(dammit.originalEncoding, 'utf-8')
+
+    def test_convert_hebrew(self):
+        hebrew = "\xed\xe5\xec\xf9"
+        dammit = UnicodeDammit(hebrew, ["iso-8859-8"])
+        self.assertEquals(dammit.originalEncoding, 'iso-8859-8')
+        self.assertEquals(dammit.unicode, u'\u05dd\u05d5\u05dc\u05e9')
+
+    def test_dont_see_smart_quotes_where_there_are_none(self):
+        utf_8 = "\343\202\261\343\203\274\343\202\277\343\202\244 Watch"
+        dammit = UnicodeDammit(utf_8)
+        self.assertEquals(dammit.originalEncoding, 'utf-8')
+        self.assertEquals(dammit.unicode.encode("utf-8"), utf_8)
+
+    def test_ignore_inappropriate_codecs(self):
+        utf8_data = u"Räksmörgås".encode("utf-8")
+        dammit = UnicodeDammit(utf8_data, ["iso-8859-8"])
+        self.assertEquals(dammit.originalEncoding, 'utf-8')
+
+    def test_ignore_invalid_codecs(self):
+        utf8_data = u"Räksmörgås".encode("utf-8")
+        for bad_encoding in ['.utf8', '...', 'utF---16.!']:
+            dammit = UnicodeDammit(utf8_data, [bad_encoding])
+            self.assertEquals(dammit.originalEncoding, 'utf-8')