Defer to html5lib's Unicode converter rather than using Unicode, Dammit. The lxml treebuilder still uses UD.

3 files changed, 33 insertions, 1 deletions

diff --git a/beautifulsoup/builder/__init__.py b/beautifulsoup/builder/__init__.py
index cf5e6c6..5bf5929 100644
--- a/beautifulsoup/builder/__init__.py
+++ b/beautifulsoup/builder/__init__.py
@@ -25,6 +25,10 @@ class TreeBuilder(Entities):
     def feed(self, markup):
         raise NotImplementedError()
 
+    def prepare_markup(self, markup, user_specified_encoding=None,
+                       document_declared_encoding=None):
+        return markup, None, None
+
     def test_fragment_to_document(self, fragment):
         """Wrap an HTML fragment to make it look like a document.
 
diff --git a/beautifulsoup/builder/html5lib_builder.py b/beautifulsoup/builder/html5lib_builder.py
index dc95493..95151da 100644
--- a/beautifulsoup/builder/html5lib_builder.py
+++ b/beautifulsoup/builder/html5lib_builder.py
@@ -13,10 +13,23 @@ from beautifulsoup.element import (
 class HTML5TreeBuilder(HTMLTreeBuilder):
     """Use html5lib to build a tree."""
 
+    def prepare_markup(self, markup, user_specified_encoding):
+        # Store the user-specified encoding for use later on.
+        self.user_specified_encoding = user_specified_encoding
+        return markup, None, None
+
     # These methods are defined by Beautiful Soup.
     def feed(self, markup):
         parser = html5lib.HTMLParser(tree=self.create_treebuilder)
-        doc = parser.parse(markup)
+        doc = parser.parse(markup, encoding=self.user_specified_encoding)
+
+        # Set the character encoding detected by the tokenizer.
+        if isinstance(markup, unicode):
+            # We need to special-case this because html5lib sets
+            # charEncoding to UTF-8 if it gets Unicode input.
+            doc.originalEncoding = None
+        else:
+            doc.originalEncoding = parser.tokenizer.stream.charEncoding[0]
 
     def create_treebuilder(self, namespaceHTMLElements):
         self.underlying_builder = TreeBuilderForHtml5lib(
diff --git a/beautifulsoup/builder/lxml_builder.py b/beautifulsoup/builder/lxml_builder.py
index 9ced9f0..a1f8c1e 100644
--- a/beautifulsoup/builder/lxml_builder.py
+++ b/beautifulsoup/builder/lxml_builder.py
@@ -1,6 +1,7 @@
 from lxml import etree
 from beautifulsoup.element import Comment, Doctype
 from beautifulsoup.builder import HTMLTreeBuilder
+from beautifulsoup.dammit import UnicodeDammit
 
 class LXMLTreeBuilder(HTMLTreeBuilder):
 
@@ -11,6 +12,20 @@ class LXMLTreeBuilder(HTMLTreeBuilder):
         self.parser = parser_class(target=self)
         self.soup = None
 
+    def prepare_markup(self, markup, user_specified_encoding=None,
+                       document_declared_encoding=None):
+        """
+        :return: A 3-tuple (markup, original encoding, encoding
+        declared within markup).
+        """
+        if isinstance(markup, unicode):
+            return markup, None, None
+
+        try_encodings = [user_specified_encoding, document_declared_encoding]
+        dammit = UnicodeDammit(markup, try_encodings, isHTML=True)
+        return dammit.markup, dammit.originalEncoding, dammit.declaredHTMLEncoding
+
+
     def feed(self, markup):
         self.parser.feed(markup)
         self.parser.close()