From ddf9d04e42168fdb25b742b35efc891789a4b6c9 Mon Sep 17 00:00:00 2001 From: Leonard Richardson Date: Fri, 18 Feb 2011 11:09:59 -0500 Subject: Have the html5lib builder set the sniffed encoding after parsing, rather than before as happens with lxml. --- beautifulsoup/builder/html5lib_builder.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'beautifulsoup/builder/html5lib_builder.py') diff --git a/beautifulsoup/builder/html5lib_builder.py b/beautifulsoup/builder/html5lib_builder.py index dc95493..a5aaa01 100644 --- a/beautifulsoup/builder/html5lib_builder.py +++ b/beautifulsoup/builder/html5lib_builder.py @@ -18,6 +18,9 @@ class HTML5TreeBuilder(HTMLTreeBuilder): parser = html5lib.HTMLParser(tree=self.create_treebuilder) doc = parser.parse(markup) + # Set the character encoding detected by the tokenizer. + doc.originalEncoding = parser.tokenizer.stream.charEncoding[0] + def create_treebuilder(self, namespaceHTMLElements): self.underlying_builder = TreeBuilderForHtml5lib( self.soup, namespaceHTMLElements) -- cgit v1.2.3 From 75c5891980c961dfe36745c1934010560666f938 Mon Sep 17 00:00:00 2001 From: Leonard Richardson Date: Fri, 18 Feb 2011 11:29:43 -0500 Subject: Pass the user-specified encoding in to html5lib rather than dropping it on the floor. --- beautifulsoup/builder/html5lib_builder.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'beautifulsoup/builder/html5lib_builder.py') diff --git a/beautifulsoup/builder/html5lib_builder.py b/beautifulsoup/builder/html5lib_builder.py index a5aaa01..bb0e374 100644 --- a/beautifulsoup/builder/html5lib_builder.py +++ b/beautifulsoup/builder/html5lib_builder.py @@ -13,10 +13,15 @@ from beautifulsoup.element import ( class HTML5TreeBuilder(HTMLTreeBuilder): """Use html5lib to build a tree.""" + def prepare_markup(self, markup, user_specified_encoding): + # Store the user-specified encoding for use later on. + self.user_specified_encoding = user_specified_encoding + return markup, None, None + # These methods are defined by Beautiful Soup. def feed(self, markup): parser = html5lib.HTMLParser(tree=self.create_treebuilder) - doc = parser.parse(markup) + doc = parser.parse(markup, encoding=self.user_specified_encoding) # Set the character encoding detected by the tokenizer. doc.originalEncoding = parser.tokenizer.stream.charEncoding[0] -- cgit v1.2.3 From ac15de23301583df65b09ea1e34062aaa80de5dd Mon Sep 17 00:00:00 2001 From: Leonard Richardson Date: Fri, 18 Feb 2011 11:36:01 -0500 Subject: Don't let html5lib set the original encoding to UTF-8 if the input was actually Unicode. --- beautifulsoup/builder/html5lib_builder.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'beautifulsoup/builder/html5lib_builder.py') diff --git a/beautifulsoup/builder/html5lib_builder.py b/beautifulsoup/builder/html5lib_builder.py index bb0e374..95151da 100644 --- a/beautifulsoup/builder/html5lib_builder.py +++ b/beautifulsoup/builder/html5lib_builder.py @@ -24,7 +24,12 @@ class HTML5TreeBuilder(HTMLTreeBuilder): doc = parser.parse(markup, encoding=self.user_specified_encoding) # Set the character encoding detected by the tokenizer. - doc.originalEncoding = parser.tokenizer.stream.charEncoding[0] + if isinstance(markup, unicode): + # We need to special-case this because html5lib sets + # charEncoding to UTF-8 if it gets Unicode input. + doc.originalEncoding = None + else: + doc.originalEncoding = parser.tokenizer.stream.charEncoding[0] def create_treebuilder(self, namespaceHTMLElements): self.underlying_builder = TreeBuilderForHtml5lib( -- cgit v1.2.3