diff options
| -rw-r--r-- | beautifulsoup/__init__.py | 28 | ||||
| -rw-r--r-- | beautifulsoup/builder/__init__.py | 4 | ||||
| -rw-r--r-- | beautifulsoup/builder/html5lib_builder.py | 15 | ||||
| -rw-r--r-- | beautifulsoup/builder/lxml_builder.py | 15 | ||||
| -rw-r--r-- | beautifulsoup/dammit.py | 11 | ||||
| -rw-r--r-- | tests/test_html5lib.py | 17 | ||||
| -rw-r--r-- | tests/test_lxml.py | 56 | ||||
| -rw-r--r-- | tests/test_soup.py | 61 |
8 files changed, 182 insertions, 25 deletions
diff --git a/beautifulsoup/__init__.py b/beautifulsoup/__init__.py index f2c20de..32ea73f 100644 --- a/beautifulsoup/__init__.py +++ b/beautifulsoup/__init__.py @@ -144,37 +144,29 @@ class BeautifulStoneSoup(Tag): self.builder.soup = self self.parseOnlyThese = parseOnlyThese - self.fromEncoding = fromEncoding self.reset() if hasattr(markup, 'read'): # It's a file-type object. markup = markup.read() - self.markup = markup + self.markup, self.originalEncoding, self.declaredHTMLEncoding = ( + self.builder.prepare_markup(markup, fromEncoding)) + try: - self._feed(isHTML=self.builder.assume_html) + self._feed() except StopParsing: pass - self.markup = None # The markup can now be GCed. + + # Clear out the markup and the builder so they can be CGed. + self.markup = None self.builder.soup = None - self.builder = None # So can the builder. + self.builder = None - def _feed(self, inDocumentEncoding=None, isHTML=False): + def _feed(self): # Convert the document to Unicode. - markup = self.markup - if isinstance(markup, unicode): - if not hasattr(self, 'originalEncoding'): - self.originalEncoding = None - else: - dammit = UnicodeDammit\ - (markup, [self.fromEncoding, inDocumentEncoding], - isHTML=isHTML) - markup = dammit.unicode - self.originalEncoding = dammit.originalEncoding - self.declaredHTMLEncoding = dammit.declaredHTMLEncoding self.builder.reset() - self.builder.feed(markup) + self.builder.feed(self.markup) # Close out any unfinished strings and close all the open tags. self.endData() while self.currentTag.name != self.ROOT_TAG_NAME: diff --git a/beautifulsoup/builder/__init__.py b/beautifulsoup/builder/__init__.py index cf5e6c6..5bf5929 100644 --- a/beautifulsoup/builder/__init__.py +++ b/beautifulsoup/builder/__init__.py @@ -25,6 +25,10 @@ class TreeBuilder(Entities): def feed(self, markup): raise NotImplementedError() + def prepare_markup(self, markup, user_specified_encoding=None, + document_declared_encoding=None): + return markup, None, None + def test_fragment_to_document(self, fragment): """Wrap an HTML fragment to make it look like a document. diff --git a/beautifulsoup/builder/html5lib_builder.py b/beautifulsoup/builder/html5lib_builder.py index dc95493..95151da 100644 --- a/beautifulsoup/builder/html5lib_builder.py +++ b/beautifulsoup/builder/html5lib_builder.py @@ -13,10 +13,23 @@ from beautifulsoup.element import ( class HTML5TreeBuilder(HTMLTreeBuilder): """Use html5lib to build a tree.""" + def prepare_markup(self, markup, user_specified_encoding): + # Store the user-specified encoding for use later on. + self.user_specified_encoding = user_specified_encoding + return markup, None, None + # These methods are defined by Beautiful Soup. def feed(self, markup): parser = html5lib.HTMLParser(tree=self.create_treebuilder) - doc = parser.parse(markup) + doc = parser.parse(markup, encoding=self.user_specified_encoding) + + # Set the character encoding detected by the tokenizer. + if isinstance(markup, unicode): + # We need to special-case this because html5lib sets + # charEncoding to UTF-8 if it gets Unicode input. + doc.originalEncoding = None + else: + doc.originalEncoding = parser.tokenizer.stream.charEncoding[0] def create_treebuilder(self, namespaceHTMLElements): self.underlying_builder = TreeBuilderForHtml5lib( diff --git a/beautifulsoup/builder/lxml_builder.py b/beautifulsoup/builder/lxml_builder.py index 9ced9f0..a1f8c1e 100644 --- a/beautifulsoup/builder/lxml_builder.py +++ b/beautifulsoup/builder/lxml_builder.py @@ -1,6 +1,7 @@ from lxml import etree from beautifulsoup.element import Comment, Doctype from beautifulsoup.builder import HTMLTreeBuilder +from beautifulsoup.dammit import UnicodeDammit class LXMLTreeBuilder(HTMLTreeBuilder): @@ -11,6 +12,20 @@ class LXMLTreeBuilder(HTMLTreeBuilder): self.parser = parser_class(target=self) self.soup = None + def prepare_markup(self, markup, user_specified_encoding=None, + document_declared_encoding=None): + """ + :return: A 3-tuple (markup, original encoding, encoding + declared within markup). + """ + if isinstance(markup, unicode): + return markup, None, None + + try_encodings = [user_specified_encoding, document_declared_encoding] + dammit = UnicodeDammit(markup, try_encodings, isHTML=True) + return dammit.markup, dammit.originalEncoding, dammit.declaredHTMLEncoding + + def feed(self, markup): self.parser.feed(markup) self.parser.close() diff --git a/beautifulsoup/dammit.py b/beautifulsoup/dammit.py index 78bd4b2..954ca54 100644 --- a/beautifulsoup/dammit.py +++ b/beautifulsoup/dammit.py @@ -58,13 +58,12 @@ class UnicodeDammit: return u = None - for proposedEncoding in overrideEncodings: - u = self._convertFrom(proposedEncoding) - if u: break - if not u: - for proposedEncoding in (documentEncoding, sniffedEncoding): + for proposedEncoding in ( + overrideEncodings + [documentEncoding, sniffedEncoding]): + if proposedEncoding is not None: u = self._convertFrom(proposedEncoding) - if u: break + if u: + break # If no luck and we have auto-detection library, try that: if not u and chardet and not isinstance(self.markup, unicode): diff --git a/tests/test_html5lib.py b/tests/test_html5lib.py index 3efdebf..59d84a3 100644 --- a/tests/test_html5lib.py +++ b/tests/test_html5lib.py @@ -3,6 +3,7 @@ from beautifulsoup.element import Comment from test_lxml import ( TestLXMLBuilder, TestLXMLBuilderInvalidMarkup, + TestLXMLBuilderEncodingConversion, ) class TestHTML5Builder(TestLXMLBuilder): @@ -138,3 +139,19 @@ class TestHTML5BuilderInvalidMarkup(TestLXMLBuilderInvalidMarkup): utf8 = utf8.replace("\xe9", "\xc3\xa9") #print soup + + +class TestHTML5LibEncodingConversion(TestLXMLBuilderEncodingConversion): + @property + def default_builder(self): + return HTML5TreeBuilder() + + def test_real_hebrew_document(self): + # A real-world test to make sure we can convert ISO-8859-9 (a + # Hebrew encoding) to UTF-8. + soup = self.soup(self.HEBREW_DOCUMENT, + fromEncoding="iso-8859-8") + self.assertEquals(soup.originalEncoding, 'iso8859-8') + self.assertEquals( + soup.encode('utf-8'), + self.HEBREW_DOCUMENT.decode("iso-8859-8").encode("utf-8")) diff --git a/tests/test_lxml.py b/tests/test_lxml.py index b002227..4c11b1d 100644 --- a/tests/test_lxml.py +++ b/tests/test_lxml.py @@ -375,3 +375,59 @@ class TestLXMLBuilderInvalidMarkup(SoupTest): #CDATA sections are ignored. markup = "<div><![CDATA[foo]]>" self.assertSoupEquals(markup, "<div></div>") + + +class TestLXMLBuilderEncodingConversion(SoupTest): + # Test Beautiful Soup's ability to decode and encode from various + # encodings. + + def setUp(self): + super(TestLXMLBuilderEncodingConversion, self).setUp() + self.unicode_data = u"<html><head></head><body><foo>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</foo></body></html>" + self.utf8_data = self.unicode_data.encode("utf-8") + # Just so you know what it looks like. + self.assertEqual( + self.utf8_data, + "<html><head></head><body><foo>Sacr\xc3\xa9 bleu!</foo></body></html>") + + def test_ascii_in_unicode_out(self): + # ASCII input is converted to Unicode. The originalEncoding + # attribute is set. + ascii = "<foo>a</foo>" + soup_from_ascii = self.soup(ascii) + unicode_output = soup_from_ascii.decode() + self.assertTrue(isinstance(unicode_output, unicode)) + self.assertEquals(unicode_output, self.document_for(ascii)) + self.assertEquals(soup_from_ascii.originalEncoding, "ascii") + + def test_unicode_in_unicode_out(self): + # Unicode input is left alone. The originalEncoding attribute + # is not set. + soup_from_unicode = self.soup(self.unicode_data) + self.assertEquals(soup_from_unicode.decode(), self.unicode_data) + self.assertEquals(soup_from_unicode.foo.string, u'Sacr\xe9 bleu!') + self.assertEquals(soup_from_unicode.originalEncoding, None) + + def test_utf8_in_unicode_out(self): + # UTF-8 input is converted to Unicode. The originalEncoding + # attribute is set. + soup_from_utf8 = self.soup(self.utf8_data) + self.assertEquals(soup_from_utf8.decode(), self.unicode_data) + self.assertEquals(soup_from_utf8.foo.string, u'Sacr\xe9 bleu!') + + def test_utf8_out(self): + # The internal data structures can be encoded as UTF-8. + soup_from_unicode = self.soup(self.unicode_data) + self.assertEquals(soup_from_unicode.encode('utf-8'), self.utf8_data) + + HEBREW_DOCUMENT = '<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>' + + def test_real_hebrew_document(self): + # A real-world test to make sure we can convert ISO-8859-9 (a + # Hebrew encoding) to UTF-8. + soup = self.soup(self.HEBREW_DOCUMENT, + fromEncoding="iso-8859-8") + self.assertEquals(soup.originalEncoding, 'iso-8859-8') + self.assertEquals( + soup.encode('utf-8'), + self.HEBREW_DOCUMENT.decode("iso-8859-8").encode("utf-8")) diff --git a/tests/test_soup.py b/tests/test_soup.py new file mode 100644 index 0000000..4fb2142 --- /dev/null +++ b/tests/test_soup.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +"""Tests of Beautiful Soup as a whole.""" + +import unittest +from beautifulsoup.element import SoupStrainer +from beautifulsoup.dammit import UnicodeDammit +from beautifulsoup.testing import SoupTest + + +class TestSelectiveParsing(SoupTest): + + def test_parse_with_soupstrainer(self): + markup = "No<b>Yes</b><a>No<b>Yes <c>Yes</c></b>" + strainer = SoupStrainer("b") + soup = self.soup(markup, parseOnlyThese=strainer) + self.assertEquals(soup.encode(), "<b>Yes</b><b>Yes <c>Yes</c></b>") + + +class TestUnicodeDammit(unittest.TestCase): + """Standalone tests of Unicode, Dammit.""" + + def test_smart_quotes_to_xml_entities(self): + markup = "<foo>\x91\x92\x93\x94</foo>" + dammit = UnicodeDammit(markup) + self.assertEquals( + dammit.unicode, "<foo>‘’“”</foo>") + + def test_smart_quotes_to_html_entities(self): + markup = "<foo>\x91\x92\x93\x94</foo>" + dammit = UnicodeDammit(markup, smartQuotesTo="html") + self.assertEquals( + dammit.unicode, "<foo>‘’“”</foo>") + + def test_detect_utf8(self): + utf8 = "\xc3\xa9" + dammit = UnicodeDammit(utf8) + self.assertEquals(dammit.unicode, u'\xe9') + self.assertEquals(dammit.originalEncoding, 'utf-8') + + def test_convert_hebrew(self): + hebrew = "\xed\xe5\xec\xf9" + dammit = UnicodeDammit(hebrew, ["iso-8859-8"]) + self.assertEquals(dammit.originalEncoding, 'iso-8859-8') + self.assertEquals(dammit.unicode, u'\u05dd\u05d5\u05dc\u05e9') + + def test_dont_see_smart_quotes_where_there_are_none(self): + utf_8 = "\343\202\261\343\203\274\343\202\277\343\202\244 Watch" + dammit = UnicodeDammit(utf_8) + self.assertEquals(dammit.originalEncoding, 'utf-8') + self.assertEquals(dammit.unicode.encode("utf-8"), utf_8) + + def test_ignore_inappropriate_codecs(self): + utf8_data = u"Räksmörgås".encode("utf-8") + dammit = UnicodeDammit(utf8_data, ["iso-8859-8"]) + self.assertEquals(dammit.originalEncoding, 'utf-8') + + def test_ignore_invalid_codecs(self): + utf8_data = u"Räksmörgås".encode("utf-8") + for bad_encoding in ['.utf8', '...', 'utF---16.!']: + dammit = UnicodeDammit(utf8_data, [bad_encoding]) + self.assertEquals(dammit.originalEncoding, 'utf-8') |