Defer to html5lib's Unicode converter rather than using Unicode, Dammit. The lxml treebuilder still uses UD.

1 files changed, 5 insertions, 6 deletions

diff --git a/beautifulsoup/dammit.py b/beautifulsoup/dammit.py
index 78bd4b2..954ca54 100644
--- a/beautifulsoup/dammit.py
+++ b/beautifulsoup/dammit.py
@@ -58,13 +58,12 @@ class UnicodeDammit:
             return
 
         u = None
-        for proposedEncoding in overrideEncodings:
-            u = self._convertFrom(proposedEncoding)
-            if u: break
-        if not u:
-            for proposedEncoding in (documentEncoding, sniffedEncoding):
+        for proposedEncoding in (
+            overrideEncodings + [documentEncoding, sniffedEncoding]):
+            if proposedEncoding is not None:
                 u = self._convertFrom(proposedEncoding)
-                if u: break
+                if u:
+                    break
 
         # If no luck and we have auto-detection library, try that:
         if not u and chardet and not isinstance(self.markup, unicode):