From ac15de23301583df65b09ea1e34062aaa80de5dd Mon Sep 17 00:00:00 2001 From: Leonard Richardson Date: Fri, 18 Feb 2011 11:36:01 -0500 Subject: Don't let html5lib set the original encoding to UTF-8 if the input was actually Unicode. --- beautifulsoup/builder/html5lib_builder.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'beautifulsoup/builder/html5lib_builder.py') diff --git a/beautifulsoup/builder/html5lib_builder.py b/beautifulsoup/builder/html5lib_builder.py index bb0e374..95151da 100644 --- a/beautifulsoup/builder/html5lib_builder.py +++ b/beautifulsoup/builder/html5lib_builder.py @@ -24,7 +24,12 @@ class HTML5TreeBuilder(HTMLTreeBuilder): doc = parser.parse(markup, encoding=self.user_specified_encoding) # Set the character encoding detected by the tokenizer. - doc.originalEncoding = parser.tokenizer.stream.charEncoding[0] + if isinstance(markup, unicode): + # We need to special-case this because html5lib sets + # charEncoding to UTF-8 if it gets Unicode input. + doc.originalEncoding = None + else: + doc.originalEncoding = parser.tokenizer.stream.charEncoding[0] def create_treebuilder(self, namespaceHTMLElements): self.underlying_builder = TreeBuilderForHtml5lib( -- cgit v1.2.3