diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/test_html5lib.py | 17 | ||||
| -rw-r--r-- | tests/test_lxml.py | 56 | ||||
| -rw-r--r-- | tests/test_soup.py | 61 |
3 files changed, 134 insertions, 0 deletions
diff --git a/tests/test_html5lib.py b/tests/test_html5lib.py index 3efdebf..59d84a3 100644 --- a/tests/test_html5lib.py +++ b/tests/test_html5lib.py @@ -3,6 +3,7 @@ from beautifulsoup.element import Comment from test_lxml import ( TestLXMLBuilder, TestLXMLBuilderInvalidMarkup, + TestLXMLBuilderEncodingConversion, ) class TestHTML5Builder(TestLXMLBuilder): @@ -138,3 +139,19 @@ class TestHTML5BuilderInvalidMarkup(TestLXMLBuilderInvalidMarkup): utf8 = utf8.replace("\xe9", "\xc3\xa9") #print soup + + +class TestHTML5LibEncodingConversion(TestLXMLBuilderEncodingConversion): + @property + def default_builder(self): + return HTML5TreeBuilder() + + def test_real_hebrew_document(self): + # A real-world test to make sure we can convert ISO-8859-9 (a + # Hebrew encoding) to UTF-8. + soup = self.soup(self.HEBREW_DOCUMENT, + fromEncoding="iso-8859-8") + self.assertEquals(soup.originalEncoding, 'iso8859-8') + self.assertEquals( + soup.encode('utf-8'), + self.HEBREW_DOCUMENT.decode("iso-8859-8").encode("utf-8")) diff --git a/tests/test_lxml.py b/tests/test_lxml.py index b002227..4c11b1d 100644 --- a/tests/test_lxml.py +++ b/tests/test_lxml.py @@ -375,3 +375,59 @@ class TestLXMLBuilderInvalidMarkup(SoupTest): #CDATA sections are ignored. markup = "<div><![CDATA[foo]]>" self.assertSoupEquals(markup, "<div></div>") + + +class TestLXMLBuilderEncodingConversion(SoupTest): + # Test Beautiful Soup's ability to decode and encode from various + # encodings. + + def setUp(self): + super(TestLXMLBuilderEncodingConversion, self).setUp() + self.unicode_data = u"<html><head></head><body><foo>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</foo></body></html>" + self.utf8_data = self.unicode_data.encode("utf-8") + # Just so you know what it looks like. + self.assertEqual( + self.utf8_data, + "<html><head></head><body><foo>Sacr\xc3\xa9 bleu!</foo></body></html>") + + def test_ascii_in_unicode_out(self): + # ASCII input is converted to Unicode. The originalEncoding + # attribute is set. + ascii = "<foo>a</foo>" + soup_from_ascii = self.soup(ascii) + unicode_output = soup_from_ascii.decode() + self.assertTrue(isinstance(unicode_output, unicode)) + self.assertEquals(unicode_output, self.document_for(ascii)) + self.assertEquals(soup_from_ascii.originalEncoding, "ascii") + + def test_unicode_in_unicode_out(self): + # Unicode input is left alone. The originalEncoding attribute + # is not set. + soup_from_unicode = self.soup(self.unicode_data) + self.assertEquals(soup_from_unicode.decode(), self.unicode_data) + self.assertEquals(soup_from_unicode.foo.string, u'Sacr\xe9 bleu!') + self.assertEquals(soup_from_unicode.originalEncoding, None) + + def test_utf8_in_unicode_out(self): + # UTF-8 input is converted to Unicode. The originalEncoding + # attribute is set. + soup_from_utf8 = self.soup(self.utf8_data) + self.assertEquals(soup_from_utf8.decode(), self.unicode_data) + self.assertEquals(soup_from_utf8.foo.string, u'Sacr\xe9 bleu!') + + def test_utf8_out(self): + # The internal data structures can be encoded as UTF-8. + soup_from_unicode = self.soup(self.unicode_data) + self.assertEquals(soup_from_unicode.encode('utf-8'), self.utf8_data) + + HEBREW_DOCUMENT = '<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>' + + def test_real_hebrew_document(self): + # A real-world test to make sure we can convert ISO-8859-9 (a + # Hebrew encoding) to UTF-8. + soup = self.soup(self.HEBREW_DOCUMENT, + fromEncoding="iso-8859-8") + self.assertEquals(soup.originalEncoding, 'iso-8859-8') + self.assertEquals( + soup.encode('utf-8'), + self.HEBREW_DOCUMENT.decode("iso-8859-8").encode("utf-8")) diff --git a/tests/test_soup.py b/tests/test_soup.py new file mode 100644 index 0000000..4fb2142 --- /dev/null +++ b/tests/test_soup.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +"""Tests of Beautiful Soup as a whole.""" + +import unittest +from beautifulsoup.element import SoupStrainer +from beautifulsoup.dammit import UnicodeDammit +from beautifulsoup.testing import SoupTest + + +class TestSelectiveParsing(SoupTest): + + def test_parse_with_soupstrainer(self): + markup = "No<b>Yes</b><a>No<b>Yes <c>Yes</c></b>" + strainer = SoupStrainer("b") + soup = self.soup(markup, parseOnlyThese=strainer) + self.assertEquals(soup.encode(), "<b>Yes</b><b>Yes <c>Yes</c></b>") + + +class TestUnicodeDammit(unittest.TestCase): + """Standalone tests of Unicode, Dammit.""" + + def test_smart_quotes_to_xml_entities(self): + markup = "<foo>\x91\x92\x93\x94</foo>" + dammit = UnicodeDammit(markup) + self.assertEquals( + dammit.unicode, "<foo>‘’“”</foo>") + + def test_smart_quotes_to_html_entities(self): + markup = "<foo>\x91\x92\x93\x94</foo>" + dammit = UnicodeDammit(markup, smartQuotesTo="html") + self.assertEquals( + dammit.unicode, "<foo>‘’“”</foo>") + + def test_detect_utf8(self): + utf8 = "\xc3\xa9" + dammit = UnicodeDammit(utf8) + self.assertEquals(dammit.unicode, u'\xe9') + self.assertEquals(dammit.originalEncoding, 'utf-8') + + def test_convert_hebrew(self): + hebrew = "\xed\xe5\xec\xf9" + dammit = UnicodeDammit(hebrew, ["iso-8859-8"]) + self.assertEquals(dammit.originalEncoding, 'iso-8859-8') + self.assertEquals(dammit.unicode, u'\u05dd\u05d5\u05dc\u05e9') + + def test_dont_see_smart_quotes_where_there_are_none(self): + utf_8 = "\343\202\261\343\203\274\343\202\277\343\202\244 Watch" + dammit = UnicodeDammit(utf_8) + self.assertEquals(dammit.originalEncoding, 'utf-8') + self.assertEquals(dammit.unicode.encode("utf-8"), utf_8) + + def test_ignore_inappropriate_codecs(self): + utf8_data = u"Räksmörgås".encode("utf-8") + dammit = UnicodeDammit(utf8_data, ["iso-8859-8"]) + self.assertEquals(dammit.originalEncoding, 'utf-8') + + def test_ignore_invalid_codecs(self): + utf8_data = u"Räksmörgås".encode("utf-8") + for bad_encoding in ['.utf8', '...', 'utF---16.!']: + dammit = UnicodeDammit(utf8_data, [bad_encoding]) + self.assertEquals(dammit.originalEncoding, 'utf-8') |