Merged from trunk.

16 files changed, 864 insertions, 610 deletions

diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..fe99812
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,5 @@
+include *.txt
+include doc/Makefile
+include doc/source/*.py
+include doc/source/*.rst
+include doc/source/*.jpg
diff --git a/NEWS.txt b/NEWS.txt
index 8f16cd5..54caf6a 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -7,6 +7,13 @@
 * Issue a warning if characters were replaced with REPLACEMENT
   CHARACTER during Unicode conversion.
 
+* Restored compatibility with Python 2.6.
+
+* The install process no longer installs docs or auxillary text files.
+
+* It's now possible to deepcopy a BeautifulSoup object created with
+  Python's built-in HTML parser.
+
 = 4.0.0b6 (20110216) =
 
 * Multi-valued attributes like "class" always have a list of values,
diff --git a/README.txt b/README.txt
index a09824f..305c51e 100644
--- a/README.txt
+++ b/README.txt
@@ -42,10 +42,11 @@ documentation.
 
 = Running the unit tests =
 
-Beautiful Soup supports unit test discovery. You can run the tests
-from the project root directory with this command:
+Beautiful Soup supports unit test discovery from the project root directory:
 
- $ python -m unittest discover -s bs4
+ $ nosetests
+
+ $ python -m unittest discover -s bs4 # Python 2.7 and up
 
 If you checked out the source tree, you should see a script in the
 home directory called test-all-versions. This script will run the unit
diff --git a/TODO.txt b/TODO.txt
index 8b239b1..61f9aee 100644
--- a/TODO.txt
+++ b/TODO.txt
@@ -1,10 +1,6 @@
 Bugs
 ----
 
-* Issue a DataLossWarning if Unicode, Dammit replaces characters.
-
-* I think whitespace may not be processed correctly.
-
 * html5lib doesn't support SoupStrainers, which is OK, but there
   should be a warning about it.
 
diff --git a/bs4/__init__.py b/bs4/__init__.py
index 786b57b..647a815 100644
--- a/bs4/__init__.py
+++ b/bs4/__init__.py
@@ -17,7 +17,7 @@ http://www.crummy.com/software/BeautifulSoup/bs4/doc/
 """
 
 __author__ = "Leonard Richardson (leonardr@segfault.org)"
-__version__ = "4.0.0b6"
+__version__ = "4.0.0b7"
 __copyright__ = "Copyright (c) 2004-2012 Leonard Richardson"
 __license__ = "MIT"
 
diff --git a/bs4/builder/_htmlparser.py b/bs4/builder/_htmlparser.py
index ec6d456..64dfb27 100644
--- a/bs4/builder/_htmlparser.py
+++ b/bs4/builder/_htmlparser.py
@@ -38,35 +38,7 @@ from bs4.builder import (
 
 HTMLPARSER = 'html.parser'
 
-class HTMLParserTreeBuilder(HTMLParser, HTMLTreeBuilder):
-
-    is_xml = False
-    features = [HTML, STRICT, HTMLPARSER]
-
-    def __init__(self, *args, **kwargs):
-        if CONSTRUCTOR_TAKES_STRICT:
-            kwargs['strict'] = False
-        return super(HTMLParserTreeBuilder, self).__init__(*args, **kwargs)
-
-    def prepare_markup(self, markup, user_specified_encoding=None,
-                       document_declared_encoding=None):
-        """
-        :return: A 4-tuple (markup, original encoding, encoding
-        declared within markup, whether any characters had to be
-        replaced with REPLACEMENT CHARACTER).
-        """
-        if isinstance(markup, unicode):
-            return markup, None, None, False
-
-        try_encodings = [user_specified_encoding, document_declared_encoding]
-        dammit = UnicodeDammit(markup, try_encodings, is_html=True)
-        return (dammit.markup, dammit.original_encoding,
-                dammit.declared_html_encoding,
-                dammit.contains_replacement_characters)
-
-    def feed(self, markup):
-        super(HTMLParserTreeBuilder, self).feed(markup)
-
+class BeautifulSoupHTMLParser(HTMLParser):
     def handle_starttag(self, name, attrs):
         # XXX namespace
         self.soup.handle_starttag(name, None, dict(attrs))
@@ -81,9 +53,15 @@ class HTMLParserTreeBuilder(HTMLParser, HTMLTreeBuilder):
         # XXX workaround for a bug in HTMLParser. Remove this once
         # it's fixed.
         if name.startswith('x'):
-            data = unichr(int(name.lstrip('x'), 16))
+            real_name = int(name.lstrip('x'), 16)
         else:
-            data = unichr(int(name))
+            real_name = int(name)
+
+        try:
+            data = unichr(real_name)
+        except (ValueError, OverflowError), e:
+            data = u"\N{REPLACEMENT CHARACTER}"
+
         self.handle_data(data)
 
     def handle_entityref(self, name):
@@ -121,6 +99,40 @@ class HTMLParserTreeBuilder(HTMLParser, HTMLTreeBuilder):
         self.soup.handle_data(data)
         self.soup.endData(ProcessingInstruction)
 
+
+class HTMLParserTreeBuilder(HTMLTreeBuilder):
+
+    is_xml = False
+    features = [HTML, STRICT, HTMLPARSER]
+
+    def __init__(self, *args, **kwargs):
+        if CONSTRUCTOR_TAKES_STRICT:
+            kwargs['strict'] = False
+        self.parser_args = (args, kwargs)
+
+    def prepare_markup(self, markup, user_specified_encoding=None,
+                       document_declared_encoding=None):
+        """
+        :return: A 4-tuple (markup, original encoding, encoding
+        declared within markup, whether any characters had to be
+        replaced with REPLACEMENT CHARACTER).
+        """
+        if isinstance(markup, unicode):
+            return markup, None, None, False
+
+        try_encodings = [user_specified_encoding, document_declared_encoding]
+        dammit = UnicodeDammit(markup, try_encodings, is_html=True)
+        return (dammit.markup, dammit.original_encoding,
+                dammit.declared_html_encoding,
+                dammit.contains_replacement_characters)
+
+    def feed(self, markup):
+        args, kwargs = self.parser_args
+        parser = BeautifulSoupHTMLParser(*args, **kwargs)
+        parser.soup = self.soup
+        parser.feed(markup)
+
+
 # Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some
 # 3.2.3 code. This ensures they don't treat markup like <p></p> as a
 # string.
@@ -147,7 +159,7 @@ if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT:
    )*
   \s*                                # trailing whitespace
 """, re.VERBOSE)
-    HTMLParserTreeBuilder.locatestarttagend = locatestarttagend
+    BeautifulSoupHTMLParser.locatestarttagend = locatestarttagend
 
     from html.parser import tagfind, attrfind
 
@@ -210,7 +222,7 @@ if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT:
         self.cdata_elem = elem.lower()
         self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I)
 
-    HTMLParserTreeBuilder.parse_starttag = parse_starttag
-    HTMLParserTreeBuilder.set_cdata_mode = set_cdata_mode
+    BeautifulSoupHTMLParser.parse_starttag = parse_starttag
+    BeautifulSoupHTMLParser.set_cdata_mode = set_cdata_mode
 
     CONSTRUCTOR_TAKES_STRICT = True
diff --git a/bs4/element.py b/bs4/element.py
index 926fb8f..653bb05 100644
--- a/bs4/element.py
+++ b/bs4/element.py
@@ -765,7 +765,7 @@ class Tag(PageElement):
         # Turn the data structure into Unicode, then encode the
         # Unicode.
         u = self.decode(indent_level, encoding, formatter)
-        return u.encode(encoding, errors=errors)
+        return u.encode(encoding, errors)
 
     def decode(self, indent_level=None,
                eventual_encoding=DEFAULT_OUTPUT_ENCODING,
diff --git a/bs4/testing.py b/bs4/testing.py
index c374a29..e5e8c93 100644
--- a/bs4/testing.py
+++ b/bs4/testing.py
@@ -1,16 +1,22 @@
 """Helper classes for tests."""
 
+import functools
 import unittest
+from unittest import TestCase
 from bs4 import BeautifulSoup
 from bs4.element import Comment, SoupStrainer
-from bs4.builder import LXMLTreeBuilder
-
+try:
+    from bs4.builder import LXMLTreeBuilder
+    default_builder = LXMLTreeBuilder
+except ImportError, e:
+    from bs4.builder import HTMLParserTreeBuilder
+    default_builder = HTMLParserTreeBuilder
 
 class SoupTest(unittest.TestCase):
 
     @property
     def default_builder(self):
-        return LXMLTreeBuilder()
+        return default_builder()
 
     def soup(self, markup, **kwargs):
         """Build a Beautiful Soup object from markup."""
@@ -31,3 +37,15 @@ class SoupTest(unittest.TestCase):
             compare_parsed_to = to_parse
 
         self.assertEqual(obj.decode(), self.document_for(compare_parsed_to))
+
+def skipIf(condition, reason):
+   def nothing(test, *args, **kwargs):
+       return None
+
+   def decorator(test_item):
+       if condition:
+           return nothing
+       else:
+           return test_item
+
+   return decorator
diff --git a/bs4/tests/test_builder_registry.py b/bs4/tests/test_builder_registry.py
index 5f60462..92ad10f 100644
--- a/bs4/tests/test_builder_registry.py
+++ b/bs4/tests/test_builder_registry.py
@@ -6,8 +6,6 @@ from bs4 import BeautifulSoup
 from bs4.builder import (
     builder_registry as registry,
     HTMLParserTreeBuilder,
-    LXMLTreeBuilderForXML,
-    LXMLTreeBuilder,
     TreeBuilderRegistry,
 )
 
@@ -18,7 +16,10 @@ except ImportError:
     HTML5LIB_PRESENT = False
 
 try:
-    from bs4.builder import LXMLTreeBuilder
+    from bs4.builder import (
+        LXMLTreeBuilderForXML,
+        LXMLTreeBuilder,
+        )
     LXML_PRESENT = True
 except ImportError:
     LXML_PRESENT = False
@@ -28,10 +29,13 @@ class BuiltInRegistryTest(unittest.TestCase):
     """Test the built-in registry with the default builders registered."""
 
     def test_combination(self):
-        self.assertEqual(registry.lookup('fast', 'html'),
-                          LXMLTreeBuilder)
-        self.assertEqual(registry.lookup('permissive', 'xml'),
-                          LXMLTreeBuilderForXML)
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('fast', 'html'),
+                             LXMLTreeBuilder)
+
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('permissive', 'xml'),
+                             LXMLTreeBuilderForXML)
         self.assertEqual(registry.lookup('strict', 'html'),
                           HTMLParserTreeBuilder)
         if HTML5LIB_PRESENT:
@@ -41,15 +45,20 @@ class BuiltInRegistryTest(unittest.TestCase):
     def test_lookup_by_markup_type(self):
         if LXML_PRESENT:
             self.assertEqual(registry.lookup('html'), LXMLTreeBuilder)
+            self.assertEqual(registry.lookup('xml'), LXMLTreeBuilderForXML)
         else:
-            self.assertEqual(registry.lookup('html'), HTML5TreeBuilder)
-        self.assertEqual(registry.lookup('xml'), LXMLTreeBuilderForXML)
+            self.assertEqual(registry.lookup('xml'), None)
+            if HTML5LIB_PRESENT:
+                self.assertEqual(registry.lookup('html'), HTML5TreeBuilder)
+            else:
+                self.assertEqual(registry.lookup('html'), HTMLParserTreeBuilder)
 
     def test_named_library(self):
-        self.assertEqual(registry.lookup('lxml', 'xml'),
-                          LXMLTreeBuilderForXML)
-        self.assertEqual(registry.lookup('lxml', 'html'),
-                          LXMLTreeBuilder)
+        if LXML_PRESENT:
+            self.assertEqual(registry.lookup('lxml', 'xml'),
+                             LXMLTreeBuilderForXML)
+            self.assertEqual(registry.lookup('lxml', 'html'),
+                             LXMLTreeBuilder)
         if HTML5LIB_PRESENT:
             self.assertEqual(registry.lookup('html5lib'),
                               HTML5TreeBuilder)
diff --git a/bs4/tests/test_docs.py b/bs4/tests/test_docs.py
new file mode 100644
index 0000000..b7b427d
--- /dev/null
+++ b/bs4/tests/test_docs.py
@@ -0,0 +1,36 @@
+"Test harness for doctests."
+
+# pylint: disable-msg=E0611,W0142
+
+__metaclass__ = type
+__all__ = [
+    'additional_tests',
+    ]
+
+import atexit
+import doctest
+import os
+from pkg_resources import (
+    resource_filename, resource_exists, resource_listdir, cleanup_resources)
+import unittest
+
+DOCTEST_FLAGS = (
+    doctest.ELLIPSIS |
+    doctest.NORMALIZE_WHITESPACE |
+    doctest.REPORT_NDIFF)
+
+
+# def additional_tests():
+#     "Run the doc tests (README.txt and docs/*, if any exist)"
+#     doctest_files = [
+#         os.path.abspath(resource_filename('bs4', 'README.txt'))]
+#     if resource_exists('bs4', 'docs'):
+#         for name in resource_listdir('bs4', 'docs'):
+#             if name.endswith('.txt'):
+#                 doctest_files.append(
+#                     os.path.abspath(
+#                         resource_filename('bs4', 'docs/%s' % name)))
+#     kwargs = dict(module_relative=False, optionflags=DOCTEST_FLAGS)
+#     atexit.register(cleanup_resources)
+#     return unittest.TestSuite((
+#         doctest.DocFileSuite(*doctest_files, **kwargs)))
diff --git a/bs4/tests/test_html5lib.py b/bs4/tests/test_html5lib.py
index d972b2d..3f00d52 100644
--- a/bs4/tests/test_html5lib.py
+++ b/bs4/tests/test_html5lib.py
@@ -4,17 +4,14 @@ try:
 except ImportError, e:
     HTML5LIB_PRESENT = False
 from bs4.element import Comment, SoupStrainer
-from test_lxml import (
-    TestLXMLBuilder,
-    TestLXMLBuilderInvalidMarkup,
-    TestLXMLBuilderEncodingConversion,
-    )
+import test_htmlparser
 import unittest
+from bs4.testing import skipIf
 
-@unittest.skipIf(
+@skipIf(
     not HTML5LIB_PRESENT,
     "html5lib seems not to be present, not testing its tree builder.")
-class TestHTML5Builder(TestLXMLBuilder):
+class TestHTML5Builder(test_htmlparser.TestHTMLParserTreeBuilder):
     """See `BuilderSmokeTest`."""
 
     @property
@@ -34,7 +31,7 @@ class TestHTML5Builder(TestLXMLBuilder):
         # A bare string is turned into some kind of HTML document or
         # fragment recognizable as the original string.
         #
-        # In this case, lxml puts a <p> tag around the bare string.
+        # In this case, html5lib puts a <p> tag around the bare string.
         self.assertSoupEquals(
             "A bare string", "A bare string")
 
@@ -81,10 +78,35 @@ class TestHTML5Builder(TestLXMLBuilder):
         # get a CData object.
         self.assertSoupEquals(markup, "<svg><!--[CDATA[foobar]]--></svg>")
 
-@unittest.skipIf(
+    def test_entities_in_attribute_values_converted_during_parsing(self):
+
+        # The numeric entity is recognized even without the closing
+        # semicolon.
+        text = '<x t="pi&#241ata">'
+        expected = u"pi\N{LATIN SMALL LETTER N WITH TILDE}ata"
+        soup = self.soup(text)
+        self.assertEqual(soup.x['t'], expected)
+
+    def test_naked_ampersands(self):
+        # Ampersands are not treated as entities, unlike in html.parser.
+        text = "<p>AT&T</p>"
+        soup = self.soup(text)
+        self.assertEqual(soup.p.string, "AT&T")
+
+    def test_namespaced_system_doctype(self):
+        # Test a namespaced doctype with a system id.
+        self._test_doctype('xsl:stylesheet SYSTEM "htmlent.dtd"')
+
+    def test_namespaced_public_doctype(self):
+        # Test a namespaced doctype with a public id.
+        self._test_doctype('xsl:stylesheet PUBLIC "htmlent.dtd"')
+
+
+@skipIf(
     not HTML5LIB_PRESENT,
     "html5lib seems not to be present, not testing it on invalid markup.")
-class TestHTML5BuilderInvalidMarkup(TestLXMLBuilderInvalidMarkup):
+class TestHTML5BuilderInvalidMarkup(
+    test_htmlparser.TestHTMLParserTreeBuilderInvalidMarkup):
     """See `BuilderInvalidMarkupSmokeTest`."""
 
     @property
@@ -99,6 +121,29 @@ class TestHTML5BuilderInvalidMarkup(TestLXMLBuilderInvalidMarkup):
             '<blockquote><p><b>Foo</blockquote><p>Bar',
             '<blockquote><p><b>Foo</b></p></blockquote><p><b>Bar</b></p>')
 
+    def test_attribute_value_never_got_closed(self):
+        markup = '<a href="http://foo.com/</a> and blah and blah'
+        soup = self.soup(markup)
+        self.assertEqual(
+            soup.a['href'], "http://foo.com/</a> and blah and blah")
+
+    def test_attribute_value_was_closed_by_subsequent_tag(self):
+        markup = """<a href="foo</a>, </a><a href="bar">baz</a>"""
+        soup = self.soup(markup)
+        # The string between the first and second quotes was interpreted
+        # as the value of the 'href' attribute.
+        self.assertEqual(soup.a['href'], 'foo</a>, </a><a href=')
+
+        #The string after the second quote (bar"), was treated as an
+        #empty attribute called bar.
+        self.assertEqual(soup.a['bar'], '')
+        self.assertEqual(soup.a.string, "baz")
+
+    def test_document_starts_with_bogus_declaration(self):
+        soup = self.soup('<! Foo ><p>a</p>')
+        # The declaration is ignored altogether.
+        self.assertEqual(soup.encode(), b"<html><body><p>a</p></body></html>")
+
     def test_table_containing_bare_markup(self):
         # Markup should be in table cells, not directly in the table.
         self.assertSoupEquals("<table><div>Foo</div></table>",
@@ -245,11 +290,55 @@ class TestHTML5BuilderInvalidMarkup(TestLXMLBuilderInvalidMarkup):
         soup = self.soup("<p>&#x1000000000000;</p>")
         self.assertEqual(soup.p.string, u"\N{REPLACEMENT CHARACTER}")
 
+    def test_incomplete_declaration(self):
+        self.assertSoupEquals('a<!b <p>c', 'a<!--b &lt;p-->c')
+
+    def test_nonsensical_declaration(self):
+        soup = self.soup('<! Foo = -8><p>a</p>')
+        self.assertEquals(
+            soup.decode(),
+            "<!-- Foo = -8--><html><head></head><body><p>a</p></body></html>")
+
+    def test_unquoted_attribute_value(self):
+        soup = self.soup('<a style={height:21px;}></a>')
+        self.assertEqual(soup.a['style'], '{height:21px;}')
+
+    def test_boolean_attribute_with_no_value(self):
+        soup = self.soup("<table><td nowrap>foo</td></table>")
+        self.assertEqual(soup.table.td['nowrap'], '')
+
+    def test_cdata_where_it_doesnt_belong(self):
+        #CDATA sections are ignored.
+        markup = "<div><![CDATA[foo]]>"
+        self.assertSoupEquals(markup, "<div><!--[CDATA[foo]]--></div>")
+
+    def test_empty_element_tag_with_contents(self):
+        self.assertSoupEquals("<br>foo</br>", "<br/>foo<br/>")
+
+    def test_fake_self_closing_tag(self):
+        # If a self-closing tag presents as a normal tag, the 'open'
+        # tag is treated as an instance of the self-closing tag and
+        # the 'close' tag is ignored.
+        self.assertSoupEquals(
+            "<item><link>http://foo.com/</link></item>",
+            "<item><link/>http://foo.com/</item>")
+
+    def test_paragraphs_containing_block_display_elements(self):
+        markup = self.soup("<p>this is the definition:"
+                           "<dl><dt>first case</dt>")
+        # The <p> tag is closed before the <dl> tag begins.
+        self.assertEqual(markup.p.contents, ["this is the definition:"])
+
+    def test_multiple_values_for_the_same_attribute(self):
+        markup = '<b b="20" a="1" b="10" a="2" a="3" a="4"></b>'
+        self.assertSoupEquals(markup, '<b a="1" b="20"></b>')
+
 
-@unittest.skipIf(
+@skipIf(
     not HTML5LIB_PRESENT,
-    "html5lib seems not to be present, not testing encoding conversion.")
-class TestHTML5LibEncodingConversion(TestLXMLBuilderEncodingConversion):
+    "html5lib seems not to be present, not testing it on encoding conversion.")
+class TestHTML5LibEncodingConversion(
+    test_htmlparser.TestHTMLParserTreeBuilderEncodingConversion):
     @property
     def default_builder(self):
         return HTML5TreeBuilder()
diff --git a/bs4/tests/test_htmlparser.py b/bs4/tests/test_htmlparser.py
index ea94363..d5b6ae1 100644
--- a/bs4/tests/test_htmlparser.py
+++ b/bs4/tests/test_htmlparser.py
@@ -1,14 +1,23 @@
+import copy
 from HTMLParser import HTMLParseError
+from bs4.element import Comment, Doctype, SoupStrainer
 from bs4.builder import HTMLParserTreeBuilder
 from bs4.element import CData
-from test_lxml import (
-    TestLXMLBuilder,
-    TestLXMLBuilderEncodingConversion,
-    TestLXMLBuilderInvalidMarkup,
-    )
+from bs4.testing import SoupTest
 
-class TestHTMLParserTreeBuilder(TestLXMLBuilder):
-    """See `BuilderSmokeTest`."""
+class TestHTMLParserTreeBuilder(SoupTest):
+
+    """A smoke test for the built-in tree builder.
+
+    Subclass this to test some other HTML tree builder. Subclasses of
+    this test ensure that all of Beautiful Soup's tree builders
+    generate more or less the same trees.
+
+    It's okay for trees to differ--just override the appropriate test
+    method to demonstrate how one tree builder differs from the
+    default builder. But in general, all HTML tree builders should
+    generate trees that make most of these tests pass.
+    """
 
     @property
     def default_builder(self):
@@ -30,23 +39,9 @@ class TestHTMLParserTreeBuilder(TestLXMLBuilder):
         self.assertEqual(string, "foobar")
         self.assertTrue(isinstance(string, CData))
 
-    # These are tests that could be 'fixed' by improving the
-    # HTMLParserTreeBuilder, but I don't think it's worth it. Users
-    # will have fewer headaches if they use one of the other tree
-    # builders.
-
-    def test_empty_element(self):
-        # HTML's empty-element tags are not recognized as such
-        # unless they are presented as empty-element tags.
-        self.assertSoupEquals(
-            "<p>A <meta> tag</p>", "<p>A <meta> tag</meta></p>")
-
-        self.assertSoupEquals(
-            "<p>Foo<br/>bar</p>", "<p>Foo<br/>bar</p>")
-
     def test_hex_entities_in_text(self):
         # XXX This tests a workaround for a bug in HTMLParser.
-        self.assertSoupEquals("&#xf1;", u"\xf1")
+        self.assertSoupEquals("<p>&#xf1;</p>", u"<p>\xf1</p>")
 
     def test_entities_in_attribute_values_converted_during_parsing(self):
 
@@ -114,14 +109,466 @@ class TestHTMLParserTreeBuilder(TestLXMLBuilder):
         self.assertRaises(HTMLParseError, self._test_doctype,
                           'xsl:stylesheet PUBLIC "htmlent.dtd"')
 
+    def _test_doctype(self, doctype_fragment):
+        """Run a battery of assertions on a given doctype string.
+
+        HTMLParser doesn't actually behave like this, so this method
+        is never called in this class. But many other builders do
+        behave like this, so I've put the method in the superclass.
+        """
+        doctype_str = '<!DOCTYPE %s>' % doctype_fragment
+        markup = doctype_str + '<p>foo</p>'
+        soup = self.soup(markup)
+        doctype = soup.contents[0]
+        self.assertEqual(doctype.__class__, Doctype)
+        self.assertEqual(doctype, doctype_fragment)
+        self.assertEqual(str(soup)[:len(doctype_str)], doctype_str)
+
+        # Make sure that the doctype was correctly associated with the
+        # parse tree and that the rest of the document parsed.
+        self.assertEqual(soup.p.contents[0], 'foo')
+
+# -------------------------
+
+    def test_mixed_case_tags(self):
+        # Mixed-case tags are folded to lowercase.
+        self.assertSoupEquals(
+            "<a><B><Cd><EFG></efg></CD></b></A>",
+            "<a><b><cd><efg></efg></cd></b></a>")
+
+
+    def test_empty_tag_thats_not_an_empty_element_tag(self):
+        # A tag that is empty but not an HTML empty-element tag
+        # is not presented as an empty-element tag.
+        self.assertSoupEquals("<p>", "<p></p>")
+
+    def test_comment(self):
+        # Comments are represented as Comment objects.
+        markup = "<p>foo<!--foobar-->baz</p>"
+        self.assertSoupEquals(markup)
+
+        soup = self.soup(markup)
+        comment = soup.find(text="foobar")
+        self.assertEqual(comment.__class__, Comment)
+
+    def test_nested_inline_elements(self):
+        # Inline tags can be nested indefinitely.
+        b_tag = "<b>Inside a B tag</b>"
+        self.assertSoupEquals(b_tag)
+
+        nested_b_tag = "<p>A <i>nested <b>tag</b></i></p>"
+        self.assertSoupEquals(nested_b_tag)
+
+        double_nested_b_tag = "<p>A <a>doubly <i>nested <b>tag</b></i></a></p>"
+        self.assertSoupEquals(nested_b_tag)
+
+    def test_nested_block_level_elements(self):
+        soup = self.soup('<blockquote><p><b>Foo</b></p></blockquote>')
+        blockquote = soup.blockquote
+        self.assertEqual(blockquote.p.b.string, 'Foo')
+        self.assertEqual(blockquote.b.string, 'Foo')
+
+    # This is a <table> tag containing another <table> tag in one of its
+    # cells.
+    TABLE_MARKUP_1 = ('<table id="1">'
+                     '<tr>'
+                     "<td>Here's another table:"
+                     '<table id="2">'
+                     '<tr><td>foo</td></tr>'
+                     '</table></td>')
+
+    def test_correctly_nested_tables(self):
+        markup = ('<table id="1">'
+                  '<tr>'
+                  "<td>Here's another table:"
+                  '<table id="2">'
+                  '<tr><td>foo</td></tr>'
+                  '</table></td>')
+
+        self.assertSoupEquals(
+            markup,
+            '<table id="1"><tr><td>Here\'s another table:'
+            '<table id="2"><tr><td>foo</td></tr></table>'
+            '</td></tr></table>')
+
+        self.assertSoupEquals(
+            "<table><thead><tr><td>Foo</td></tr></thead>"
+            "<tbody><tr><td>Bar</td></tr></tbody>"
+            "<tfoot><tr><td>Baz</td></tr></tfoot></table>")
+
+    def test_collapsed_whitespace(self):
+        """In most tags, whitespace is collapsed."""
+        self.assertSoupEquals("<p>   </p>", "<p> </p>")
+
+    def test_preserved_whitespace_in_pre_and_textarea(self):
+        """In <pre> and <textarea> tags, whitespace is preserved."""
+        self.assertSoupEquals("<pre>   </pre>")
+        self.assertSoupEquals("<textarea> woo  </textarea>")
+
+    def test_single_quote_attribute_values_become_double_quotes(self):
+        self.assertSoupEquals("<foo attr='bar'></foo>",
+                              '<foo attr="bar"></foo>')
+
+    def test_attribute_values_with_nested_quotes_are_left_alone(self):
+        text = """<foo attr='bar "brawls" happen'>a</foo>"""
+        self.assertSoupEquals(text)
+
+    def test_attribute_values_with_double_nested_quotes_get_quoted(self):
+        text = """<foo attr='bar "brawls" happen'>a</foo>"""
+        soup = self.soup(text)
+        soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
+        self.assertSoupEquals(
+            soup.foo.decode(),
+            """<foo attr="Brawls happen at &quot;Bob\'s Bar&quot;">a</foo>""")
+
+    def test_ampersand_in_attribute_value_gets_quoted(self):
+        self.assertSoupEquals('<this is="really messed up & stuff"></this>',
+                              '<this is="really messed up &amp; stuff"></this>')
+
+    def test_entities_in_strings_converted_during_parsing(self):
+        # Both XML and HTML entities are converted to Unicode characters
+        # during parsing.
+        text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
+        expected = u"<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>"
+        self.assertSoupEquals(text, expected)
+
+    def test_smart_quotes_converted_on_the_way_in(self):
+        # Microsoft smart quotes are converted to Unicode characters during
+        # parsing.
+        quote = b"<p>\x91Foo\x92</p>"
+        soup = self.soup(quote)
+        self.assertEqual(
+            soup.p.string,
+            u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
+
+    def test_non_breaking_spaces_converted_on_the_way_in(self):
+        soup = self.soup("<a>&nbsp;&nbsp;</a>")
+        self.assertEqual(soup.a.string, u"\N{NO-BREAK SPACE}" * 2)
+
+    def test_real_iso_latin_document(self):
+        # Smoke test of interrelated functionality, using an
+        # easy-to-understand document.
+
+        # Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
+        unicode_html = u'<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>'
+
+        # That's because we're going to encode it into ISO-Latin-1, and use
+        # that to test.
+        iso_latin_html = unicode_html.encode("iso-8859-1")
+
+        # Parse the ISO-Latin-1 HTML.
+        soup = self.soup(iso_latin_html)
+        # Encode it to UTF-8.
+        result = soup.encode("utf-8")
+
+        # What do we expect the result to look like? Well, it would
+        # look like unicode_html, except that the META tag would say
+        # UTF-8 instead of ISO-Latin-1.
+        expected = unicode_html.replace("ISO-Latin-1", "utf-8")
+
+        # And, of course, it would be in UTF-8, not Unicode.
+        expected = expected.encode("utf-8")
+
+        # Ta-da!
+        self.assertEqual(result, expected)
+
+    def test_real_shift_jis_document(self):
+        # Smoke test to make sure the parser can handle a document in
+        # Shift-JIS encoding, without choking.
+        shift_jis_html = (
+            b'<html><head></head><body><pre>'
+            b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f'
+            b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c'
+            b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B'
+            b'</pre></body></html>')
+        unicode_html = shift_jis_html.decode("shift-jis")
+        soup = self.soup(unicode_html)
+
+        # Make sure the parse tree is correctly encoded to various
+        # encodings.
+        self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8"))
+        self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp"))
+
+    # Tests below this line need work.
+
+    def test_meta_tag_reflects_current_encoding(self):
+        # Here's the <meta> tag saying that a document is
+        # encoded in Shift-JIS.
+        meta_tag = ('<meta content="text/html; charset=x-sjis" '
+                    'http-equiv="Content-type"/>')
+
+        # Here's a document incorporating that meta tag.
+        shift_jis_html = (
+            '<html><head>\n%s\n'
+            '<meta http-equiv="Content-language" content="ja"/>'
+            '</head><body>Shift-JIS markup goes here.') % meta_tag
+        soup = self.soup(shift_jis_html)
+
+        # Parse the document, and the charset is replaced with a
+        # generic value.
+        parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'})
+        self.assertEqual(parsed_meta['content'],
+                          'text/html; charset=%SOUP-ENCODING%')
+        self.assertEqual(parsed_meta.contains_substitutions, True)
+
+        # For the rest of the story, see TestSubstitutions in
+        # test_tree.py.
+
+    def test_entities_converted_on_the_way_out(self):
+        text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
+        expected = u"&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;".encode("utf-8")
+        soup = self.soup(text)
+        str = soup.p.string
+        #self.assertEqual(str.encode("utf-8"), expected)
+
+    def test_br_tag_is_empty_element(self):
+        """A <br> tag is designated as an empty-element tag."""
+        soup = self.soup("<br></br>")
+        self.assertTrue(soup.br.is_empty_element)
+        self.assertEqual(str(soup.br), "<br/>")
 
-class TestHTMLParserTreeBuilderInvalidMarkup(TestLXMLBuilderInvalidMarkup):
-    # Oddly enough, HTMLParser seems to handle invalid markup exactly
-    # the same as lxml.
-    pass
+    def test_p_tag_is_not_empty_element(self):
+        """A <p> tag is not designated as an empty-element tag."""
+        soup = self.soup("<p/>")
+        self.assertFalse(soup.p.is_empty_element)
+        self.assertEqual(str(soup.p), "<p></p>")
 
+    def test_soupstrainer(self):
+        strainer = SoupStrainer("b")
+        soup = self.soup("A <b>bold</b> <meta/> <i>statement</i>",
+                         parse_only=strainer)
+        self.assertEqual(soup.decode(), "<b>bold</b>")
 
-class TestHTMLParserTreeBuilderEncodingConversion(
-    TestLXMLBuilderEncodingConversion):
-    # Re-run the lxml tests for HTMLParser
-    pass
+    def test_deepcopy(self):
+        # Make sure you can copy the builder. This is important because
+        # the builder is part of a BeautifulSoup object, and we want to be
+        # able to copy that.
+        copy.deepcopy(self.default_builder)
+
+class TestHTMLParserTreeBuilderInvalidMarkup(SoupTest):
+    """Tests of invalid markup for the default tree builder.
+
+    Subclass this to test other builders.
+
+    These are very likely to give different results for different tree
+    builders. It's not required that a tree builder handle invalid
+    markup at all.
+    """
+
+    @property
+    def default_builder(self):
+        return HTMLParserTreeBuilder()
+
+    def test_table_containing_bare_markup(self):
+        # Markup should be in table cells, not directly in the table.
+        self.assertSoupEquals("<table><div>Foo</div></table>")
+
+    def test_incorrectly_nested_table(self):
+        # The second <table> tag is floating in the <tr> tag
+        # rather than being inside a <td>.
+        bad_markup = ('<table id="1">'
+                      '<tr>'
+                      "<td>Here's another table:</td>"
+                      '<table id="2">'
+                      '<tr><td>foo</td></tr>'
+                      '</table></td>')
+
+
+    def test_unclosed_a_tag(self):
+        # <a> tags really ought to be closed at some point.
+        #
+        # We have all the <div> tags because HTML5 says to duplicate
+        # the <a> tag rather than closing it, and that's what html5lib
+        # does.
+        markup = """<div id="1">
+ <a href="foo">
+</div>
+<div id="2">
+ <div id="3">
+   <a href="bar"></a>
+  </div>
+</div>"""
+
+        expect = """<div id="1">
+<a href="foo">
+</a></div>
+<div id="2">
+<div id="3">
+<a href="bar"></a>
+</div>
+</div>"""
+        self.assertSoupEquals(markup, expect)
+
+    def test_unclosed_block_level_elements(self):
+        # Unclosed block-level elements should be closed.
+        self.assertSoupEquals(
+            '<blockquote><p><b>Foo</blockquote><p>Bar',
+            '<blockquote><p><b>Foo</b></p></blockquote><p>Bar</p>')
+
+    def test_fake_self_closing_tag(self):
+        # If a self-closing tag presents as a normal tag, it's treated
+        # as one.
+        self.assertSoupEquals(
+            "<item><link>http://foo.com/</link></item>",
+            "<item><link>http://foo.com/</link></item>")
+
+    def test_boolean_attribute_with_no_value(self):
+        soup = self.soup("<table><td nowrap>foo</td></table>")
+        self.assertEqual(soup.table.td['nowrap'], None)
+
+    def test_incorrectly_nested_tables(self):
+        self.assertSoupEquals(
+            '<table><tr><table><tr id="nested">',
+            '<table><tr><table><tr id="nested"></tr></table></tr></table>')
+
+    def test_floating_text_in_table(self):
+        self.assertSoupEquals("<table><td></td>foo<td>bar</td></table>")
+
+    def test_paragraphs_containing_block_display_elements(self):
+        markup = self.soup("<p>this is the definition:"
+                           "<dl><dt>first case</dt>")
+        # The <p> tag is not closed before the <dl> tag begins.
+        self.assertEqual(len(markup.p.contents), 2)
+
+    def test_empty_element_tag_with_contents(self):
+        self.assertSoupEquals("<br>foo</br>", "<br>foo</br>")
+
+    def test_doctype_in_body(self):
+        markup = "<p>one<!DOCTYPE foobar>two</p>"
+        self.assertSoupEquals(markup)
+
+    def test_nonsensical_declaration(self):
+        # Declarations that don't make any sense are ignored.
+        self.assertRaises(HTMLParseError, self.soup, '<! Foo = -8><p>a</p>')
+
+    def test_whitespace_in_doctype(self):
+        # A declaration that has extra whitespace is ignored.
+        self.assertRaises(
+            HTMLParseError, self.soup,
+            '<! DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">' +
+             '<p>foo</p>')
+
+    def test_incomplete_declaration(self):
+        self.assertRaises(HTMLParseError, self.soup, 'a<!b <p>c')
+
+    def test_cdata_where_it_doesnt_belong(self):
+        #CDATA sections are ignored.
+        markup = "<div><![CDATA[foo]]>"
+        soup = self.soup(markup)
+        self.assertEquals(soup.div.contents[0], CData("foo"))
+
+    def test_attribute_value_never_got_closed(self):
+        markup = '<a href="http://foo.com/</a> and blah and blah'
+        soup = self.soup(markup)
+        self.assertEqual(soup.encode(), b"")
+
+    def test_attribute_value_with_embedded_brackets(self):
+        soup = self.soup('<a b="<a>">')
+        self.assertEqual(soup.a['b'], '<a>')
+
+    def test_nonexistent_entity(self):
+        soup = self.soup("<p>foo&#bar;baz</p>")
+        # This is very strange.
+        self.assertEqual(soup.p.string, "foo<p")
+
+        # Compare a real entity.
+        soup = self.soup("<p>foo&#100;baz</p>")
+        self.assertEqual(soup.p.string, "foodbaz")
+
+        # Also compare html5lib, which preserves the &# before the
+        # entity name.
+
+    def test_entity_out_of_range(self):
+        # An entity that's out of range will be replaced with
+        # REPLACEMENT CHARACTER.
+        soup = self.soup("<p>&#10000000000000;</p>")
+        self.assertEqual(soup.p.string, u"\N{REPLACEMENT CHARACTER}")
+
+        soup = self.soup("<p>&#x1000000000000;</p>")
+        self.assertEqual(soup.p.string, u"\N{REPLACEMENT CHARACTER}")
+
+        soup = self.soup("<p>&#1000000000;</p>")
+        self.assertEqual(soup.p.string, u"\N{REPLACEMENT CHARACTER}")
+
+
+    def test_entity_was_not_finished(self):
+        soup = self.soup("<p>&lt;Hello&gt")
+        # Compare html5lib, which completes the entity.
+        self.assertEqual(soup.p.string, "<Hello")
+
+    def test_document_ends_with_incomplete_declaration(self):
+        soup = self.soup('<p>a<!b')
+        # This becomes a string 'a'. The incomplete declaration is ignored.
+        # Compare html5lib, which turns it into a comment.
+        self.assertEqual(soup.p.contents, ['a'])
+
+    def test_document_starts_with_bogus_declaration(self):
+        self.assertRaises(HTMLParseError, self.soup, '<! Foo ><p>a</p>')
+
+    def test_tag_name_contains_unicode(self):
+        # Unicode characters in tag names are stripped.
+        tag_name = u"<our\N{SNOWMAN}>Joe</our\N{SNOWMAN}>"
+        self.assertSoupEquals("<our>Joe</our>")
+
+    def test_multiple_values_for_the_same_attribute(self):
+        markup = '<b b="20" a="1" b="10" a="2" a="3" a="4"></b>'
+        self.assertSoupEquals(markup, '<b a="4" b="10"></b>')
+
+class TestHTMLParserTreeBuilderEncodingConversion(SoupTest):
+    # Test Beautiful Soup's ability to decode and encode from various
+    # encodings.
+
+    @property
+    def default_builder(self):
+        return HTMLParserTreeBuilder()
+
+    def setUp(self):
+        super(TestHTMLParserTreeBuilderEncodingConversion, self).setUp()
+        self.unicode_data = u"<html><head></head><body><foo>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</foo></body></html>"
+        self.utf8_data = self.unicode_data.encode("utf-8")
+        # Just so you know what it looks like.
+        self.assertEqual(
+            self.utf8_data,
+            b"<html><head></head><body><foo>Sacr\xc3\xa9 bleu!</foo></body></html>")
+
+    def test_ascii_in_unicode_out(self):
+        # ASCII input is converted to Unicode. The original_encoding
+        # attribute is set.
+        ascii = b"<foo>a</foo>"
+        soup_from_ascii = self.soup(ascii)
+        unicode_output = soup_from_ascii.decode()
+        self.assertTrue(isinstance(unicode_output, unicode))
+        self.assertEqual(unicode_output, self.document_for(ascii.decode()))
+        self.assertEqual(soup_from_ascii.original_encoding, "ascii")
+
+    def test_unicode_in_unicode_out(self):
+        # Unicode input is left alone. The original_encoding attribute
+        # is not set.
+        soup_from_unicode = self.soup(self.unicode_data)
+        self.assertEqual(soup_from_unicode.decode(), self.unicode_data)
+        self.assertEqual(soup_from_unicode.foo.string, u'Sacr\xe9 bleu!')
+        self.assertEqual(soup_from_unicode.original_encoding, None)
+
+    def test_utf8_in_unicode_out(self):
+        # UTF-8 input is converted to Unicode. The original_encoding
+        # attribute is set.
+        soup_from_utf8 = self.soup(self.utf8_data)
+        self.assertEqual(soup_from_utf8.decode(), self.unicode_data)
+        self.assertEqual(soup_from_utf8.foo.string, u'Sacr\xe9 bleu!')
+
+    def test_utf8_out(self):
+        # The internal data structures can be encoded as UTF-8.
+        soup_from_unicode = self.soup(self.unicode_data)
+        self.assertEqual(soup_from_unicode.encode('utf-8'), self.utf8_data)
+
+    HEBREW_DOCUMENT = b'<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>'
+
+    def test_real_hebrew_document(self):
+        # A real-world test to make sure we can convert ISO-8859-9 (a
+        # Hebrew encoding) to UTF-8.
+        soup = self.soup(self.HEBREW_DOCUMENT,
+                         from_encoding="iso-8859-8")
+        self.assertEqual(soup.original_encoding, 'iso-8859-8')
+        self.assertEqual(
+            soup.encode('utf-8'),
+            self.HEBREW_DOCUMENT.decode("iso-8859-8").encode("utf-8"))
diff --git a/bs4/tests/test_lxml.py b/bs4/tests/test_lxml.py
index 3603528..4d19e7f 100644
--- a/bs4/tests/test_lxml.py
+++ b/bs4/tests/test_lxml.py
@@ -2,13 +2,22 @@
 
 import re
 
+try:
+    from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML
+    LXML_PRESENT = True
+except ImportError, e:
+    LXML_PRESENT = False
+
 from bs4 import BeautifulSoup
-from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML
 from bs4.element import Comment, Doctype, SoupStrainer
-from bs4.testing import SoupTest
-
-
-class TestLXMLBuilder(SoupTest):
+from bs4.testing import skipIf
+from bs4.tests import test_htmlparser
+from bs4.testing import skipIf
+
+@skipIf(
+    not LXML_PRESENT,
+    "lxml seems not to be present, not testing its tree builder.")
+class TestLXMLTreeBuilder(test_htmlparser.TestHTMLParserTreeBuilder):
     """A smoke test for the LXML tree builder.
 
     Subclass this to test some other HTML tree builder. Subclasses of
@@ -21,6 +30,10 @@ class TestLXMLBuilder(SoupTest):
     trees that make most of these tests pass.
     """
 
+    @property
+    def default_builder(self):
+        return LXMLTreeBuilder()
+
     def test_bare_string(self):
         # A bare string is turned into some kind of HTML document or
         # fragment recognizable as the original string.
@@ -29,11 +42,10 @@ class TestLXMLBuilder(SoupTest):
         self.assertSoupEquals(
             "A bare string", "<p>A bare string</p>")
 
-    def test_mixed_case_tags(self):
-        # Mixed-case tags are folded to lowercase.
-        self.assertSoupEquals(
-            "<a><B><Cd><EFG></efg></CD></b></A>",
-            "<a><b><cd><efg></efg></cd></b></a>")
+    def test_cdata_where_its_ok(self):
+        # lxml strips CDATA sections, no matter where they occur.
+        markup = "<svg><![CDATA[foobar]]>"
+        self.assertSoupEquals(markup, "<svg></svg>")
 
     def test_empty_element(self):
         # HTML's empty-element tags are recognized as such.
@@ -43,93 +55,16 @@ class TestLXMLBuilder(SoupTest):
         self.assertSoupEquals(
             "<p>Foo<br/>bar</p>", "<p>Foo<br/>bar</p>")
 
-    def test_empty_tag_thats_not_an_empty_element_tag(self):
-        # A tag that is empty but not an HTML empty-element tag
-        # is not presented as an empty-element tag.
-        self.assertSoupEquals("<p>", "<p></p>")
-
-    def test_comment(self):
-        # Comments are represented as Comment objects.
-        markup = "<p>foo<!--foobar-->baz</p>"
-        self.assertSoupEquals(markup)
-
-        soup = self.soup(markup)
-        comment = soup.find(text="foobar")
-        self.assertEqual(comment.__class__, Comment)
-
-    def test_nested_inline_elements(self):
-        # Inline tags can be nested indefinitely.
-        b_tag = "<b>Inside a B tag</b>"
-        self.assertSoupEquals(b_tag)
-
-        nested_b_tag = "<p>A <i>nested <b>tag</b></i></p>"
-        self.assertSoupEquals(nested_b_tag)
-
-        double_nested_b_tag = "<p>A <a>doubly <i>nested <b>tag</b></i></a></p>"
-        self.assertSoupEquals(nested_b_tag)
-
-    def test_nested_block_level_elements(self):
-        soup = self.soup('<blockquote><p><b>Foo</b></p></blockquote>')
-        blockquote = soup.blockquote
-        self.assertEqual(blockquote.p.b.string, 'Foo')
-        self.assertEqual(blockquote.b.string, 'Foo')
-
-    # This is a <table> tag containing another <table> tag in one of its
-    # cells.
-    TABLE_MARKUP_1 = ('<table id="1">'
-                     '<tr>'
-                     "<td>Here's another table:"
-                     '<table id="2">'
-                     '<tr><td>foo</td></tr>'
-                     '</table></td>')
-
-    def test_correctly_nested_tables(self):
-        markup = ('<table id="1">'
-                  '<tr>'
-                  "<td>Here's another table:"
-                  '<table id="2">'
-                  '<tr><td>foo</td></tr>'
-                  '</table></td>')
-
-        self.assertSoupEquals(
-            markup,
-            '<table id="1"><tr><td>Here\'s another table:'
-            '<table id="2"><tr><td>foo</td></tr></table>'
-            '</td></tr></table>')
-
-        self.assertSoupEquals(
-            "<table><thead><tr><td>Foo</td></tr></thead>"
-            "<tbody><tr><td>Bar</td></tr></tbody>"
-            "<tfoot><tr><td>Baz</td></tr></tfoot></table>")
-
-    def test_collapsed_whitespace(self):
-        """In most tags, whitespace is collapsed."""
-        self.assertSoupEquals("<p>   </p>", "<p> </p>")
-
-    def test_preserved_whitespace_in_pre_and_textarea(self):
-        """In <pre> and <textarea> tags, whitespace is preserved."""
-        self.assertSoupEquals("<pre>   </pre>")
-        self.assertSoupEquals("<textarea> woo  </textarea>")
-
-    def test_single_quote_attribute_values_become_double_quotes(self):
-        self.assertSoupEquals("<foo attr='bar'></foo>",
-                              '<foo attr="bar"></foo>')
-
-    def test_attribute_values_with_nested_quotes_are_left_alone(self):
-        text = """<foo attr='bar "brawls" happen'>a</foo>"""
-        self.assertSoupEquals(text)
-
-    def test_attribute_values_with_double_nested_quotes_get_quoted(self):
-        text = """<foo attr='bar "brawls" happen'>a</foo>"""
+    def test_naked_ampersands(self):
+        # Ampersands are left alone.
+        text = "<p>AT&T</p>"
         soup = self.soup(text)
-        soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
-        self.assertSoupEquals(
-            soup.foo.decode(),
-            """<foo attr="Brawls happen at &quot;Bob\'s Bar&quot;">a</foo>""")
+        self.assertEqual(soup.p.string, "AT&T")
 
-    def test_ampersand_in_attribute_value_gets_quoted(self):
-        self.assertSoupEquals('<this is="really messed up & stuff"></this>',
-                              '<this is="really messed up &amp; stuff"></this>')
+        # Even if they're in attribute values.
+        invalid_url = '<a href="http://example.org?a=1&b=2;3">foo</a>'
+        soup = self.soup(invalid_url)
+        self.assertEqual(soup.a['href'], "http://example.org?a=1&b=2;3")
 
     def test_literal_in_textarea(self):
         # Anything inside a <textarea> is supposed to be treated as
@@ -150,56 +85,6 @@ class TestLXMLBuilder(SoupTest):
         soup = self.soup('<script>%s</script>' % javascript)
         self.assertEqual(soup.script.string, javascript)
 
-    def test_naked_ampersands(self):
-        # Ampersands are left alone.
-        text = "<p>AT&T</p>"
-        soup = self.soup(text)
-        self.assertEqual(soup.p.string, "AT&T")
-
-        # Even if they're in attribute values.
-        invalid_url = '<a href="http://example.org?a=1&b=2;3">foo</a>'
-        soup = self.soup(invalid_url)
-        self.assertEqual(soup.a['href'], "http://example.org?a=1&b=2;3")
-
-    def test_entities_in_strings_converted_during_parsing(self):
-        # Both XML and HTML entities are converted to Unicode characters
-        # during parsing.
-        text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
-        expected = u"<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>"
-        self.assertSoupEquals(text, expected)
-
-    def test_smart_quotes_converted_on_the_way_in(self):
-        # Microsoft smart quotes are converted to Unicode characters during
-        # parsing.
-        quote = b"<p>\x91Foo\x92</p>"
-        soup = self.soup(quote)
-        self.assertEqual(
-            soup.p.string,
-            u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
-
-    def test_non_breaking_spaces_converted_on_the_way_in(self):
-        soup = self.soup("<a>&nbsp;&nbsp;</a>")
-        self.assertEqual(soup.a.string, u"\N{NO-BREAK SPACE}" * 2)
-
-    def test_cdata_where_its_ok(self):
-        # lxml strips CDATA sections, no matter where they occur.
-        markup = "<svg><![CDATA[foobar]]>"
-        self.assertSoupEquals(markup, "<svg></svg>")
-
-    def _test_doctype(self, doctype_fragment):
-        """Run a battery of assertions on a given doctype string."""
-        doctype_str = '<!DOCTYPE %s>' % doctype_fragment
-        markup = doctype_str + '<p>foo</p>'
-        soup = self.soup(markup)
-        doctype = soup.contents[0]
-        self.assertEqual(doctype.__class__, Doctype)
-        self.assertEqual(doctype, doctype_fragment)
-        self.assertEqual(str(soup)[:len(doctype_str)], doctype_str)
-
-        # Make sure that the doctype was correctly associated with the
-        # parse tree and that the rest of the document parsed.
-        self.assertEqual(soup.p.contents[0], 'foo')
-
     def test_doctype(self):
         # Test a normal HTML doctype you'll commonly see in a real document.
         self._test_doctype(
@@ -213,209 +98,45 @@ class TestLXMLBuilder(SoupTest):
         # Test a namespaced doctype with a public id.
         self._test_doctype('xsl:stylesheet PUBLIC "htmlent.dtd"')
 
-    def test_real_iso_latin_document(self):
-        # Smoke test of interrelated functionality, using an
-        # easy-to-understand document.
-
-        # Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
-        unicode_html = u'<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>'
-
-        # That's because we're going to encode it into ISO-Latin-1, and use
-        # that to test.
-        iso_latin_html = unicode_html.encode("iso-8859-1")
-
-        # Parse the ISO-Latin-1 HTML.
-        soup = self.soup(iso_latin_html)
-        # Encode it to UTF-8.
-        result = soup.encode("utf-8")
-
-        # What do we expect the result to look like? Well, it would
-        # look like unicode_html, except that the META tag would say
-        # UTF-8 instead of ISO-Latin-1.
-        expected = unicode_html.replace("ISO-Latin-1", "utf-8")
-
-        # And, of course, it would be in UTF-8, not Unicode.
-        expected = expected.encode("utf-8")
-
-        # Ta-da!
-        self.assertEqual(result, expected)
-
-    def test_real_shift_jis_document(self):
-        # Smoke test to make sure the parser can handle a document in
-        # Shift-JIS encoding, without choking.
-        shift_jis_html = (
-            b'<html><head></head><body><pre>'
-            b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f'
-            b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c'
-            b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B'
-            b'</pre></body></html>')
-        unicode_html = shift_jis_html.decode("shift-jis")
-        soup = self.soup(unicode_html)
-
-        # Make sure the parse tree is correctly encoded to various
-        # encodings.
-        self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8"))
-        self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp"))
-
-    # Tests below this line need work.
-
-    def test_meta_tag_reflects_current_encoding(self):
-        # Here's the <meta> tag saying that a document is
-        # encoded in Shift-JIS.
-        meta_tag = ('<meta content="text/html; charset=x-sjis" '
-                    'http-equiv="Content-type"/>')
-
-        # Here's a document incorporating that meta tag.
-        shift_jis_html = (
-            '<html><head>\n%s\n'
-            '<meta http-equiv="Content-language" content="ja"/>'
-            '</head><body>Shift-JIS markup goes here.') % meta_tag
-        soup = self.soup(shift_jis_html)
-
-        # Parse the document, and the charset is replaced with a
-        # generic value.
-        parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'})
-        self.assertEqual(parsed_meta['content'],
-                          'text/html; charset=%SOUP-ENCODING%')
-        self.assertEqual(parsed_meta.contains_substitutions, True)
-
-        # For the rest of the story, see TestSubstitutions in
-        # test_tree.py.
-
-    def test_entities_converted_on_the_way_out(self):
-        text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
-        expected = u"&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;".encode("utf-8")
-        soup = self.soup(text)
-        str = soup.p.string
-        #self.assertEqual(str.encode("utf-8"), expected)
-
-    def test_br_tag_is_empty_element(self):
-        """A <br> tag is designated as an empty-element tag."""
-        soup = self.soup("<br></br>")
-        self.assertTrue(soup.br.is_empty_element)
-        self.assertEqual(str(soup.br), "<br/>")
-
-    def test_p_tag_is_not_empty_element(self):
-        """A <p> tag is not designated as an empty-element tag."""
-        soup = self.soup("<p/>")
-        self.assertFalse(soup.p.is_empty_element)
-        self.assertEqual(str(soup.p), "<p></p>")
-
-    def test_soupstrainer(self):
-        strainer = SoupStrainer("b")
-        soup = self.soup("A <b>bold</b> <meta/> <i>statement</i>",
-                         parse_only=strainer)
-        self.assertEqual(soup.decode(), "<b>bold</b>")
-
-
-class TestLXMLBuilderInvalidMarkup(SoupTest):
-    """Tests of invalid markup for the LXML tree builder.
-
-    Subclass this to test other builders.
-
-    These are very likely to give different results for different tree
-    builders. It's not required that a tree builder handle invalid
-    markup at all.
-    """
-
-    def test_table_containing_bare_markup(self):
-        # Markup should be in table cells, not directly in the table.
-        self.assertSoupEquals("<table><div>Foo</div></table>")
-
-    def test_incorrectly_nested_table(self):
-        # The second <table> tag is floating in the <tr> tag
-        # rather than being inside a <td>.
-        bad_markup = ('<table id="1">'
-                      '<tr>'
-                      "<td>Here's another table:</td>"
-                      '<table id="2">'
-                      '<tr><td>foo</td></tr>'
-                      '</table></td>')
-
-
-    def test_unclosed_a_tag(self):
-        # <a> tags really ought to be closed at some point.
-        #
-        # We have all the <div> tags because HTML5 says to duplicate
-        # the <a> tag rather than closing it, and that's what html5lib
-        # does.
-        markup = """<div id="1">
- <a href="foo">
-</div>
-<div id="2">
- <div id="3">
-   <a href="bar"></a>
-  </div>
-</div>"""
-
-        expect = """<div id="1">
-<a href="foo">
-</a></div>
-<div id="2">
-<div id="3">
-<a href="bar"></a>
-</div>
-</div>"""
-        self.assertSoupEquals(markup, expect)
-
-    def test_unclosed_block_level_elements(self):
-        # Unclosed block-level elements should be closed.
-        self.assertSoupEquals(
-            '<blockquote><p><b>Foo</blockquote><p>Bar',
-            '<blockquote><p><b>Foo</b></p></blockquote><p>Bar</p>')
-
-    def test_fake_self_closing_tag(self):
-        # If a self-closing tag presents as a normal tag, the 'open'
-        # tag is treated as an instance of the self-closing tag and
-        # the 'close' tag is ignored.
-        self.assertSoupEquals(
-            "<item><link>http://foo.com/</link></item>",
-            "<item><link/>http://foo.com/</item>")
-
-    def test_boolean_attribute_with_no_value_gets_empty_value(self):
-        soup = self.soup("<table><td nowrap>foo</td></table>")
-        self.assertEqual(soup.table.td['nowrap'], '')
-
-    def test_incorrectly_nested_tables(self):
-        self.assertSoupEquals(
-            '<table><tr><table><tr id="nested">',
-            '<table><tr><table><tr id="nested"></tr></table></tr></table>')
+    def test_entities_in_attribute_values_converted_during_parsing(self):
 
-    def test_floating_text_in_table(self):
-        self.assertSoupEquals("<table><td></td>foo<td>bar</td></table>")
+        # The numeric entity isn't recognized without the closing
+        # semicolon.
+        text = '<x t="pi&#241ata">'
+        expected = u"pi\N{LATIN SMALL LETTER N WITH TILDE}ata"
+        soup = self.soup(text)
+        self.assertEqual(soup.x['t'], expected)
 
-    def test_paragraphs_containing_block_display_elements(self):
-        markup = self.soup("<p>this is the definition:"
-                           "<dl><dt>first case</dt>")
-        # The <p> tag is closed before the <dl> tag begins.
-        self.assertEqual(markup.p.contents, ["this is the definition:"])
+        text = '<x t="pi&#241;ata">'
+        expected = u"pi\N{LATIN SMALL LETTER N WITH TILDE}ata"
+        soup = self.soup(text)
+        self.assertEqual(soup.x['t'], u"pi\xf1ata")
 
-    def test_empty_element_tag_with_contents(self):
-        self.assertSoupEquals("<br>foo</br>", "<br/>foo")
+        text = '<x t="pi&#xf1;ata">'
+        soup = self.soup(text)
+        self.assertEqual(soup.x['t'], expected)
 
-    def test_doctype_in_body(self):
-        markup = "<p>one<!DOCTYPE foobar>two</p>"
-        self.assertSoupEquals(markup)
+        text = '<x t="sacr&eacute; bleu">'
+        soup = self.soup(text)
+        self.assertEqual(
+            soup.x['t'],
+            u"sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu")
 
-    def test_nonsensical_declaration(self):
-        # Declarations that don't make any sense are ignored.
-        self.assertSoupEquals('<! Foo = -8><p>a</p>', "<p>a</p>")
+        # This can cause valid HTML to become invalid.
+        valid_url = '<a href="http://example.org?a=1&amp;b=2;3">foo</a>'
+        soup = self.soup(valid_url)
+        self.assertEqual(soup.a['href'], "http://example.org?a=1&b=2;3")
 
-    def test_whitespace_in_doctype(self):
-        # A declaration that has extra whitespace is ignored.
-        self.assertSoupEquals(
-            ('<! DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">'
-             '<p>foo</p>'),
-            '<p>foo</p>')
 
-    def test_incomplete_declaration(self):
-        # An incomplete declaration will screw up the rest of the document.
-        self.assertSoupEquals('a<!b <p>c', '<p>a</p>')
+@skipIf(
+    not LXML_PRESENT,
+    "lxml seems not to be present, not testing it on invalid markup.")
+class TestLXMLTreeBuilderInvalidMarkup(
+    test_htmlparser.TestHTMLParserTreeBuilderInvalidMarkup):
 
-    def test_cdata_where_it_doesnt_belong(self):
-        #CDATA sections are ignored.
-        markup = "<div><![CDATA[foo]]>"
-        self.assertSoupEquals(markup, "<div></div>")
+    @property
+    def default_builder(self):
+        return LXMLTreeBuilder()
 
     def test_attribute_value_never_got_closed(self):
         markup = '<a href="http://foo.com/</a> and blah and blah'
@@ -435,13 +156,41 @@ class TestLXMLBuilderInvalidMarkup(SoupTest):
         self.assertEqual(soup.a['bar'], '')
         self.assertEqual(soup.a.string, "baz")
 
+    def test_document_starts_with_bogus_declaration(self):
+        soup = self.soup('<! Foo ><p>a</p>')
+        # The declaration is ignored altogether.
+        self.assertEqual(soup.encode(), b"<html><body><p>a</p></body></html>")
+
+    def test_incomplete_declaration(self):
+        # An incomplete declaration will screw up the rest of the document.
+        self.assertSoupEquals('a<!b <p>c', '<p>a</p>')
+
+    def test_nonsensical_declaration(self):
+        # Declarations that don't make any sense are ignored.
+        self.assertSoupEquals('<! Foo = -8><p>a</p>', "<p>a</p>")
+
     def test_unquoted_attribute_value(self):
         soup = self.soup('<a style={height:21px;}></a>')
         self.assertEqual(soup.a['style'], '{height:21px;}')
 
-    def test_attribute_value_with_embedded_brackets(self):
-        soup = self.soup('<a b="<a>">')
-        self.assertEqual(soup.a['b'], '<a>')
+    def test_whitespace_in_doctype(self):
+        # A declaration that has extra whitespace is ignored.
+        self.assertSoupEquals(
+            ('<! DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">'
+             '<p>foo</p>'),
+            '<p>foo</p>')
+
+    def test_boolean_attribute_with_no_value(self):
+        soup = self.soup("<table><td nowrap>foo</td></table>")
+        self.assertEqual(soup.table.td['nowrap'], '')
+
+    def test_cdata_where_it_doesnt_belong(self):
+        #CDATA sections are ignored.
+        markup = "<div><![CDATA[foo]]>"
+        self.assertSoupEquals(markup, "<div></div>")
+
+    def test_empty_element_tag_with_contents(self):
+        self.assertSoupEquals("<br>foo</br>", "<br/>foo")
 
     def test_nonexistent_entity(self):
         soup = self.soup("<p>foo&#bar;baz</p>")
@@ -454,162 +203,47 @@ class TestLXMLBuilderInvalidMarkup(SoupTest):
         # Also compare html5lib, which preserves the &# before the
         # entity name.
 
-    def test_entity_out_of_range(self):
-        # An entity that's out of range will be ignored.
-        soup = self.soup("<p>&#10000000000000;</p>")
-        self.assertEqual(soup.p.string, None)
-
-        soup = self.soup("<p>&#x1000000000000;</p>")
-        self.assertEqual(soup.p.string, None)
-
-
     def test_entity_was_not_finished(self):
         soup = self.soup("<p>&lt;Hello&gt")
         # Compare html5lib, which completes the entity.
         self.assertEqual(soup.p.string, "<Hello&gt")
 
-    def test_document_ends_with_incomplete_declaration(self):
-        soup = self.soup('<p>a<!b')
-        # This becomes a string 'a'. The incomplete declaration is ignored.
-        # Compare html5lib, which turns it into a comment.
-        self.assertEqual(soup.p.contents, ['a'])
+    def test_fake_self_closing_tag(self):
+        # If a self-closing tag presents as a normal tag, the 'open'
+        # tag is treated as an instance of the self-closing tag and
+        # the 'close' tag is ignored.
+        self.assertSoupEquals(
+            "<item><link>http://foo.com/</link></item>",
+            "<item><link/>http://foo.com/</item>")
 
-    def test_document_starts_with_bogus_declaration(self):
-        soup = self.soup('<! Foo ><p>a</p>')
-        # The declaration is ignored altogether.
-        self.assertEqual(soup.encode(), b"<html><body><p>a</p></body></html>")
+    def test_paragraphs_containing_block_display_elements(self):
+        markup = self.soup("<p>this is the definition:"
+                           "<dl><dt>first case</dt>")
+        # The <p> tag is closed before the <dl> tag begins.
+        self.assertEqual(markup.p.contents, ["this is the definition:"])
 
-    def test_tag_name_contains_unicode(self):
-        # Unicode characters in tag names are stripped.
-        tag_name = u"<our\N{SNOWMAN}>Joe</our\N{SNOWMAN}>"
-        self.assertSoupEquals("<our>Joe</our>")
+    def test_multiple_values_for_the_same_attribute(self):
+        markup = '<b b="20" a="1" b="10" a="2" a="3" a="4"></b>'
+        self.assertSoupEquals(markup, '<b a="1" b="20"></b>')
 
-class TestLXMLBuilderEncodingConversion(SoupTest):
-    # Test Beautiful Soup's ability to decode and encode from various
-    # encodings.
+    def test_entity_out_of_range(self):
+        # An entity that's out of range will be ignored.
+        soup = self.soup("<p>&#10000000000000;</p>")
+        self.assertEqual(0, len(soup.p.contents))
 
-    def setUp(self):
-        super(TestLXMLBuilderEncodingConversion, self).setUp()
-        self.unicode_data = u"<html><head></head><body><foo>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</foo></body></html>"
-        self.utf8_data = self.unicode_data.encode("utf-8")
-        # Just so you know what it looks like.
-        self.assertEqual(
-            self.utf8_data,
-            b"<html><head></head><body><foo>Sacr\xc3\xa9 bleu!</foo></body></html>")
-
-    def test_ascii_in_unicode_out(self):
-        # ASCII input is converted to Unicode. The original_encoding
-        # attribute is set.
-        ascii = b"<foo>a</foo>"
-        soup_from_ascii = self.soup(ascii)
-        unicode_output = soup_from_ascii.decode()
-        self.assertTrue(isinstance(unicode_output, unicode))
-        self.assertEqual(unicode_output, self.document_for(ascii.decode()))
-        self.assertEqual(soup_from_ascii.original_encoding, "ascii")
-
-    def test_unicode_in_unicode_out(self):
-        # Unicode input is left alone. The original_encoding attribute
-        # is not set.
-        soup_from_unicode = self.soup(self.unicode_data)
-        self.assertEqual(soup_from_unicode.decode(), self.unicode_data)
-        self.assertEqual(soup_from_unicode.foo.string, u'Sacr\xe9 bleu!')
-        self.assertEqual(soup_from_unicode.original_encoding, None)
-
-    def test_utf8_in_unicode_out(self):
-        # UTF-8 input is converted to Unicode. The original_encoding
-        # attribute is set.
-        soup_from_utf8 = self.soup(self.utf8_data)
-        self.assertEqual(soup_from_utf8.decode(), self.unicode_data)
-        self.assertEqual(soup_from_utf8.foo.string, u'Sacr\xe9 bleu!')
-
-    def test_utf8_out(self):
-        # The internal data structures can be encoded as UTF-8.
-        soup_from_unicode = self.soup(self.unicode_data)
-        self.assertEqual(soup_from_unicode.encode('utf-8'), self.utf8_data)
-
-    HEBREW_DOCUMENT = b'<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>'
-
-    def test_real_hebrew_document(self):
-        # A real-world test to make sure we can convert ISO-8859-9 (a
-        # Hebrew encoding) to UTF-8.
-        soup = self.soup(self.HEBREW_DOCUMENT,
-                         from_encoding="iso-8859-8")
-        self.assertEqual(soup.original_encoding, 'iso-8859-8')
-        self.assertEqual(
-            soup.encode('utf-8'),
-            self.HEBREW_DOCUMENT.decode("iso-8859-8").encode("utf-8"))
+        soup = self.soup("<p>&#x1000000000000;</p>")
+        self.assertEqual(0, len(soup.p.contents))
 
+        soup = self.soup("<p>&#1000000000;</p>")
+        self.assertEqual(0, len(soup.p.contents))
 
-class TestLXMLXMLBuilder(SoupTest):
-    """Test XML-specific parsing behavior.
 
-    Most of the tests use HTML as an example, since Beautiful Soup is
-    mainly an HTML parser. This test suite is a base for XML-specific
-    tree builders.
-    """
+@skipIf(
+    not LXML_PRESENT,
+    "lxml seems not to be present, not testing it on encoding conversion.")
+class TestLXMLParserTreeBuilderEncodingConversion(
+    test_htmlparser.TestHTMLParserTreeBuilderEncodingConversion):
 
     @property
     def default_builder(self):
-        return LXMLTreeBuilderForXML()
-
-    def test_mixed_case_tags(self):
-        # Mixed-case tags are *not* folded to lowercase, but the
-        # end tag is always the same case as the start tag.
-        self.assertSoupEquals(
-            "<a><B><Cd><EFG/></CD></b></A>",
-            "<a><B><Cd><EFG/></Cd></B></a>")
-
-
-    def test_cdata_becomes_text(self):
-        # LXML sends CData sections as 'data' events, so we can't
-        # create special CData objects for them. We have to use
-        # NavigableString. I would like to fix this, but it's not a
-        # very high priority.
-        markup = "<foo><![CDATA[iamcdata]]></foo>"
-        soup = self.soup(markup)
-        cdata = soup.foo.contents[0]
-        self.assertEqual(cdata.__class__.__name__, 'NavigableString')
-
-
-    def test_can_handle_invalid_xml(self):
-        self.assertSoupEquals("<a><b>", "<a><b/></a>")
-
-    def test_empty_element_tag(self):
-        soup = self.soup("<p><iamselfclosing/></p>")
-        self.assertTrue(soup.iamselfclosing.is_empty_element)
-
-    def test_self_empty_tag_treated_as_empty_element(self):
-        soup = self.soup("<p><iamclosed></iamclosed></p>")
-        self.assertTrue(soup.iamclosed.is_empty_element)
-
-    def test_self_nonempty_tag_is_not_empty_element(self):
-        soup = self.soup("<p><ihavecontents>contents</ihavecontents></p>")
-        self.assertFalse(soup.ihavecontents.is_empty_element)
-
-    def test_empty_tag_that_stops_being_empty_gets_a_closing_tag(self):
-        soup = self.soup("<bar/>")
-        self.assertTrue(soup.bar.is_empty_element)
-        soup.bar.insert(1, "Contents")
-        self.assertFalse(soup.bar.is_empty_element)
-        self.assertEqual(str(soup), self.document_for("<bar>Contents</bar>"))
-
-    def test_designated_empty_element_tag_has_no_closing_tag(self):
-        builder = LXMLTreeBuilderForXML(empty_element_tags=['bar'])
-        soup = BeautifulSoup(builder=builder, markup="<bar></bar>")
-        self.assertTrue(soup.bar.is_empty_element)
-        self.assertEqual(str(soup), self.document_for("<bar/>"))
-
-    def test_empty_tag_not_in_empty_element_tag_list_has_closing_tag(self):
-        builder = LXMLTreeBuilderForXML(empty_element_tags=['bar'])
-
-        soup = BeautifulSoup(builder=builder, markup="<foo/>")
-        self.assertFalse(soup.foo.is_empty_element)
-        self.assertEqual(str(soup), self.document_for("<foo></foo>"))
-
-    def test_designated_empty_element_tag_does_not_change_parser_behavior(self):
-        # The designated list of empty-element tags only affects how
-        # empty tags are presented. It does not affect how tags are
-        # parsed--that's the parser's job.
-        builder = LXMLTreeBuilderForXML(empty_element_tags=['bar'])
-        soup = BeautifulSoup(builder=builder, markup="<bar>contents</bar>")
-        self.assertEqual(str(soup), self.document_for("<bar>contents</bar>"))
+        return LXMLTreeBuilder()
diff --git a/bs4/tests/test_soup.py b/bs4/tests/test_soup.py
index 997a01f..896e914 100644
--- a/bs4/tests/test_soup.py
+++ b/bs4/tests/test_soup.py
@@ -182,7 +182,7 @@ class TestUnicodeDammit(unittest.TestCase):
             self.assertEqual(True, dammit.contains_replacement_characters)
             self.assertTrue(u"\ufffd" in dammit.unicode_markup)
 
-            soup = BeautifulSoup(doc)
+            soup = BeautifulSoup(doc, "html.parser")
             self.assertTrue(soup.contains_replacement_characters)
 
             msg = w[0].message
diff --git a/bs4/tests/test_tree.py b/bs4/tests/test_tree.py
index 70a7da1..f39826a 100644
--- a/bs4/tests/test_tree.py
+++ b/bs4/tests/test_tree.py
@@ -14,9 +14,18 @@ import pickle
 import re
 import warnings
 from bs4 import BeautifulSoup
-from bs4.builder import builder_registry
+from bs4.builder import (
+    builder_registry,
+    HTMLParserTreeBuilder,
+)
 from bs4.element import CData, NavigableString, SoupStrainer, Tag
-from bs4.testing import SoupTest
+from bs4.testing import (
+    SoupTest,
+    skipIf,
+)
+
+XML_BUILDER_PRESENT = (builder_registry.lookup("xml") is not None)
+LXML_PRESENT = (builder_registry.lookup("lxml") is not None)
 
 class TreeTest(SoupTest):
 
@@ -600,14 +609,15 @@ class TestTagCreation(SoupTest):
         self.assertEqual(None, new_tag.parent)
 
     def test_tag_inherits_self_closing_rules_from_builder(self):
-        xml_soup = BeautifulSoup("", "xml")
-        xml_br = xml_soup.new_tag("br")
-        xml_p = xml_soup.new_tag("p")
+        if XML_BUILDER_PRESENT:
+            xml_soup = BeautifulSoup("", "xml")
+            xml_br = xml_soup.new_tag("br")
+            xml_p = xml_soup.new_tag("p")
 
-        # Both the <br> and <p> tag are empty-element, just because
-        # they have no contents.
-        self.assertEqual(b"<br/>", xml_br.encode())
-        self.assertEqual(b"<p/>", xml_p.encode())
+            # Both the <br> and <p> tag are empty-element, just because
+            # they have no contents.
+            self.assertEqual(b"<br/>", xml_br.encode())
+            self.assertEqual(b"<p/>", xml_p.encode())
 
         html_soup = BeautifulSoup("", "html")
         html_br = html_soup.new_tag("br")
@@ -1000,10 +1010,6 @@ class TestElementObjects(SoupTest):
         markup = '<b a="1" z="5" m="3" f="2" y="4"></b>'
         self.assertSoupEquals(markup, '<b a="1" f="2" m="3" y="4" z="5"></b>')
 
-    def test_multiple_values_for_the_same_attribute_are_collapsed(self):
-        markup = '<b b="20" a="1" b="10" a="2" a="3" a="4"></b>'
-        self.assertSoupEquals(markup, '<b a="1" b="20"></b>')
-
     def test_string(self):
         # A tag that contains only a text node makes that node
         # available as .string.
diff --git a/setup.py b/setup.py
index d033def..0d5b7d7 100644
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@ except ImportError:
     from distutils.command.build_py import build_py
 
 setup(name="beautifulsoup4",
-      version = "4.0.0b6",
+      version = "4.0.0b7",
       author="Leonard Richardson",
       author_email='leonardr@segfault.org',
       url="http://www.crummy.com/software/BeautifulSoup/bs4/",
@@ -15,12 +15,6 @@ setup(name="beautifulsoup4",
       long_description="""Beautiful Soup sits atop an HTML or XML parser, providing Pythonic idioms for iterating, searching, and modifying the parse tree.""",
       license="MIT",
       packages=['bs4', 'bs4.builder', 'bs4.tests'],
-      package_data={
-        "bs4": ["doc/source/*.jpg", "doc/source/*.rst",
-                "doc/source/*.py", "doc/Makefile"]
-        },
-      data_files=[
-        ('', ['COPYING.txt', 'NEWS.txt', 'AUTHORS.txt', 'README.txt'])],
       cmdclass = {'build_py':build_py},
       classifiers=["Development Status :: 4 - Beta",
                    "Intended Audience :: Developers",