From f0b1b4f0f3c2ce4e140ba01a6690ddb16bf0fb17 Mon Sep 17 00:00:00 2001 From: Leonard Richardson Date: Fri, 28 Jan 2011 21:08:24 -0500 Subject: Made .string a dynamic property. --- TODO | 5 - beautifulsoup/__init__.py | 7 - beautifulsoup/element.py | 17 + tests/test_tree.py | 770 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 787 insertions(+), 12 deletions(-) create mode 100644 tests/test_tree.py diff --git a/TODO b/TODO index 208638d..71ff3fe 100644 --- a/TODO +++ b/TODO @@ -2,11 +2,6 @@ html5lib has its own Unicode, Dammit-like system. Converting the input to Unicode should be up to the builder. The lxml builder would use Unicode, Dammit, and the html5lib builder would be a no-op. -Calculate tag.string dynamically rather than when creating the -tree. The html5lib builder doesn't use popTag, and adding/removing -things from the tree after the fact may also change the -value/availability of tag.string. - --- Here are some unit tests that fail with HTMLParser. diff --git a/beautifulsoup/__init__.py b/beautifulsoup/__init__.py index 8817164..4a7e18b 100644 --- a/beautifulsoup/__init__.py +++ b/beautifulsoup/__init__.py @@ -194,13 +194,6 @@ class BeautifulStoneSoup(Tag): def popTag(self): tag = self.tagStack.pop() - # Tags with just one string-owning child get the child as a - # 'string' property, so that soup.tag.string is shorthand for - # soup.tag.contents[0] - if len(self.currentTag.contents) == 1 and \ - isinstance(self.currentTag.contents[0], NavigableString): - self.currentTag.string = self.currentTag.contents[0] - #print "Pop", tag.name if self.tagStack: self.currentTag = self.tagStack[-1] diff --git a/beautifulsoup/element.py b/beautifulsoup/element.py index 7649b4c..bd9bcbf 100644 --- a/beautifulsoup/element.py +++ b/beautifulsoup/element.py @@ -434,6 +434,23 @@ class Tag(PageElement, Entities): else: self.attrs = map(convert, attrs) + @property + def string(self): + """Convenience property to get the single string within this tag. + + :Return: If this tag has a single string child, return value + is that string. If this tag has no children, or more than one + child, return value is None. If this tag has one child tag, + return value is the 'string' attribute of the child tag, + recursively. + """ + if len(self.contents) != 1: + return None + child = self.contents[0] + if isinstance(child, NavigableString): + return child + return child.string + def get(self, key, default=None): """Returns the value of the 'key' attribute for the tag, or the value given for 'default' if it doesn't have that diff --git a/tests/test_tree.py b/tests/test_tree.py new file mode 100644 index 0000000..a3c4b3b --- /dev/null +++ b/tests/test_tree.py @@ -0,0 +1,770 @@ +# -*- coding: utf-8 -*- +"""Tests for Beautiful Soup's tree traversal methods. + +The tree traversal methods are the main advantage of using Beautiful +Soup over other parsers. + +Different parsers will build different Beautiful Soup trees given the +same markup, but all Beautiful Soup trees can be traversed with the +methods tested here. +""" + +import re +from beautifulsoup import BeautifulSoup +from beautifulsoup.element import SoupStrainer, Tag +from beautifulsoup.testing import SoupTest + +class TreeTest(SoupTest): + + def assertSelects(self, tags, should_match): + """Make sure that the given tags have the correct text. + + This is used in tests that define a bunch of tags, each + containing a single string, and then select certain strings by + some mechanism. + """ + self.assertEqual([tag.string for tag in tags], should_match) + + def assertSelectsIDs(self, tags, should_match): + """Make sure that the given tags have the correct IDs. + + This is used in tests that define a bunch of tags, each + containing a single string, and then select certain strings by + some mechanism. + """ + self.assertEqual([tag['id'] for tag in tags], should_match) + + +class TestFind(TreeTest): + """Basic tests of the find() method. + + find() just calls findAll() with limit=1, so it's not tested all + that thouroughly here. + """ + + def test_find_tag(self): + soup = self.soup("1234") + self.assertEqual(soup.find("b").string, "2") + + def test_unicode_text_find(self): + soup = self.soup(u'

Räksmörgås

') + self.assertEqual(soup.find(text=u'Räksmörgås'), u'Räksmörgås') + + +class TestFindAll(TreeTest): + """Basic tests of the findAll() method.""" + + def test_find_all_text_nodes(self): + """You can search the tree for text nodes.""" + soup = self.soup("Foobar\xbb") + # Exact match. + self.assertEqual(soup.findAll(text="bar"), [u"bar"]) + # Match any of a number of strings. + self.assertEqual( + soup.findAll(text=["Foo", "bar"]), [u"Foo", u"bar"]) + # Match a regular expression. + self.assertEqual(soup.findAll(text=re.compile('.*')), + [u"Foo", u"bar", u'\xbb']) + # Match anything. + self.assertEqual(soup.findAll(text=True), + [u"Foo", u"bar", u'\xbb']) + + def test_find_all_limit(self): + """You can limit the number of items returned by findAll.""" + soup = self.soup("12345") + self.assertSelects(soup.findAll('a', limit=3), ["1", "2", "3"]) + self.assertSelects(soup.findAll('a', limit=1), ["1"]) + self.assertSelects( + soup.findAll('a', limit=10), ["1", "2", "3", "4", "5"]) + + # A limit of 0 means no limit. + self.assertSelects( + soup.findAll('a', limit=0), ["1", "2", "3", "4", "5"]) + +class TestFindAllByName(TreeTest): + """Test ways of finding tags by tag name.""" + + def setUp(self): + super(TreeTest, self).setUp() + self.tree = self.soup("""First tag. + Second tag. + Third Nested tag. tag.""") + + def test_find_all_by_tag_name(self): + # Find all the tags. + self.assertSelects( + self.tree.findAll('a'), ['First tag.', 'Nested tag.']) + + def test_find_all_on_non_root_element(self): + # You can call find_all on any node, not just the root. + self.assertSelects(self.tree.c.findAll('a'), ['Nested tag.']) + + def test_calling_element_invokes_find_all(self): + self.assertSelects(self.tree('a'), ['First tag.', 'Nested tag.']) + + def test_find_all_by_tag_strainer(self): + self.assertSelects( + self.tree.findAll(SoupStrainer('a')), + ['First tag.', 'Nested tag.']) + + def test_find_all_by_tag_names(self): + self.assertSelects( + self.tree.findAll(['a', 'b']), + ['First tag.', 'Second tag.', 'Nested tag.']) + + def test_find_all_by_tag_dict(self): + self.assertSelects( + self.tree.findAll({'a' : True, 'b' : True}), + ['First tag.', 'Second tag.', 'Nested tag.']) + + def test_find_all_by_tag_re(self): + self.assertSelects( + self.tree.findAll(re.compile('^[ab]$')), + ['First tag.', 'Second tag.', 'Nested tag.']) + + def test_find_all_with_tags_matching_method(self): + # You can define an oracle method that determines whether + # a tag matches the search. + def id_matches_name(tag): + return tag.name == tag.get('id') + + tree = self.soup("""Match 1. + Does not match. + Match 2.""") + + self.assertSelects( + tree.findAll(id_matches_name), ["Match 1.", "Match 2."]) + + +class TestFindAllByAttribute(TreeTest): + + def test_find_all_by_attribute_name(self): + # You can pass in keyword arguments to findAll to search by + # attribute. + tree = self.soup(""" + Matching a. + + Non-matching Matching b.a. + """) + self.assertSelects(tree.findAll(id='first'), + ["Matching a.", "Matching b."]) + + def test_find_all_by_attribute_dict(self): + # You can pass in a dictionary as the argument 'attrs'. This + # lets you search for attributes like 'name' (a fixed argument + # to findAll) and 'class' (a reserved word in Python.) + tree = self.soup(""" + Name match. + Class match. + Non-match. + A tag called 'name1'. + """) + + # This doesn't do what you want. + self.assertSelects(tree.findAll(name='name1'), + ["A tag called 'name1'."]) + # This does what you want. + self.assertSelects(tree.findAll(attrs={'name' : 'name1'}), + ["Name match."]) + + # Passing class='class2' would cause a syntax error. + self.assertSelects(tree.findAll(attrs={'class' : 'class2'}), + ["Class match."]) + + def test_find_all_by_class(self): + # Passing in a string to 'attrs' will search the CSS class. + tree = self.soup(""" + Class 1. + Class 2. + Class 1. + """) + self.assertSelects(tree.findAll('a', '1'), ['Class 1.']) + self.assertSelects(tree.findAll(attrs='1'), ['Class 1.', 'Class 1.']) + + def test_find_all_by_attribute_soupstrainer(self): + tree = self.soup(""" + Match. + Non-match.""") + + strainer = SoupStrainer(attrs={'id' : 'first'}) + self.assertSelects(tree.findAll(strainer), ['Match.']) + + def test_find_all_with_missing_atribute(self): + # You can pass in None as the value of an attribute to findAll. + # This will match tags that do not have that attribute set. + tree = self.soup("""ID present. + No ID present. + ID is empty.""") + self.assertSelects(tree.findAll('a', id=None), ["No ID present."]) + + def test_find_all_with_defined_attribute(self): + # You can pass in None as the value of an attribute to findAll. + # This will match tags that have that attribute set to any value. + tree = self.soup("""ID present. + No ID present. + ID is empty.""") + self.assertSelects( + tree.findAll(id=True), ["ID present.", "ID is empty."]) + + def test_find_all_with_numeric_attribute(self): + # If you search for a number, it's treated as a string. + tree = self.soup("""Unquoted attribute. + Quoted attribute.""") + + expected = ["Unquoted attribute.", "Quoted attribute."] + self.assertSelects(tree.findAll(id=1), expected) + self.assertSelects(tree.findAll(id="1"), expected) + + def test_find_all_with_list_attribute_values(self): + # You can pass a list of attribute values instead of just one, + # and you'll get tags that match any of the values. + tree = self.soup("""1 + 2 + 3 + No ID.""") + self.assertSelects(tree.findAll(id=["1", "3", "4"]), + ["1", "3"]) + + def test_find_all_with_regular_expression_attribute_value(self): + # You can pass a regular expression as an attribute value, and + # you'll get tags whose values for that attribute match the + # regular expression. + tree = self.soup("""One a. + Two as. + Mixed as and bs. + One b. + No ID.""") + + self.assertSelects(tree.findAll(id=re.compile("^a+$")), + ["One a.", "Two as."]) + + +class TestParentOperations(TreeTest): + """Test navigation and searching through an element's parents.""" + + def setUp(self): + super(TestParentOperations, self).setUp() + self.tree = self.soup('''
    +