diff options
| -rw-r--r-- | bs4/builder/_lxml.py | 5 | ||||
| -rw-r--r-- | bs4/diagnose.py | 84 | ||||
| -rw-r--r-- | bs4/tests/test_lxml.py | 1 |
3 files changed, 79 insertions, 11 deletions
diff --git a/bs4/builder/_lxml.py b/bs4/builder/_lxml.py index f718ed1..be35d70 100644 --- a/bs4/builder/_lxml.py +++ b/bs4/builder/_lxml.py @@ -3,6 +3,7 @@ __all__ = [ 'LXMLTreeBuilder', ] +from io import BytesIO from StringIO import StringIO import collections from lxml import etree @@ -75,7 +76,9 @@ class LXMLTreeBuilderForXML(TreeBuilder): dammit.contains_replacement_characters) def feed(self, markup): - if isinstance(markup, basestring): + if isinstance(markup, bytes): + markup = BytesIO(markup) + elif isinstance(markup, unicode): markup = StringIO(markup) # Call feed() at least once, even if the markup is empty, # or the parser won't be initialized. diff --git a/bs4/diagnose.py b/bs4/diagnose.py index 69f739c..4b5f6e4 100644 --- a/bs4/diagnose.py +++ b/bs4/diagnose.py @@ -4,6 +4,8 @@ from HTMLParser import HTMLParser from bs4 import BeautifulSoup, __version__ from bs4.builder import builder_registry import os +import random +import time import traceback import sys @@ -70,32 +72,36 @@ def lxml_trace(data, html=True): class AnnouncingParser(HTMLParser): """Announces HTMLParser parse events, without doing anything else.""" + + def _p(self, s): + print(s) + def handle_starttag(self, name, attrs): - print "%s START" % name + self._p("%s START" % name) def handle_endtag(self, name): - print "%s END" % name + self._p("%s END" % name) def handle_data(self, data): - print "%s DATA" % data + self._p("%s DATA" % data) def handle_charref(self, name): - print "%s CHARREF" % name + self._p("%s CHARREF" % name) def handle_entityref(self, name): - print "%s ENTITYREF" % name + self._p("%s ENTITYREF" % name) def handle_comment(self, data): - print "%s COMMENT" % data + self._p("%s COMMENT" % data) def handle_decl(self, data): - print "%s DECL" % data + self._p("%s DECL" % data) def unknown_decl(self, data): - print "%s UNKNOWN-DECL" % data + self._p("%s UNKNOWN-DECL" % data) def handle_pi(self, data): - print "%s PI" % data + self._p("%s PI" % data) def htmlparser_trace(data): """Print out the HTMLParser events that occur during parsing. @@ -106,5 +112,65 @@ def htmlparser_trace(data): parser = AnnouncingParser() parser.feed(data) +_vowels = "aeiou" +_consonants = "bcdfghjklmnpqrstvwxyz" + +def rword(length=5): + "Generate a random word-like string." + s = '' + for i in range(length): + if i % 2 == 0: + t = _consonants + else: + t = _vowels + s += random.choice(t) + return s + +def rsentence(length=4): + "Generate a random sentence-like string." + return " ".join(rword(random.randint(4,9)) for i in range(length)) + +def rdoc(num_elements=1000): + """Randomly generate an invalid HTML document.""" + tag_names = ['p', 'div', 'span', 'i', 'b', 'script', 'table'] + elements = [] + for i in range(num_elements): + choice = random.randint(0,3) + if choice == 0: + # New tag. + tag_name = random.choice(tag_names) + elements.append("<%s>" % tag_name) + elif choice == 1: + elements.append(rsentence(random.randint(1,4))) + elif choice == 2: + # Close a tag. + tag_name = random.choice(tag_names) + elements.append("</%s>" % tag_name) + return "<html>" + "\n".join(elements) + "</html>" + +def benchmark_parsers(num_elements=100000): + """Very basic head-to-head performance benchmark.""" + print "Comparative parser benchmark on Beautiful Soup %s" % __version__ + data = rdoc(num_elements) + print "Generated a large invalid HTML document (%d bytes)." % len(data) + for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]: + success = False + try: + a = time.time() + soup = BeautifulSoup(data, ) + b = time.time() + success = True + except Exception, e: + print "%s could not parse the markup." % parser + traceback.print_exc() + if success: + print "BS4+%s parsed the markup in %.2fs." % (parser, b-a) + + from lxml import etree + a = time.time() + etree.HTML(data) + b = time.time() + print "Raw lxml parsed the markup in %.2fs." % (b-a) + if __name__ == '__main__': diagnose(sys.stdin.read()) diff --git a/bs4/tests/test_lxml.py b/bs4/tests/test_lxml.py index f32fc2b..39e4bd4 100644 --- a/bs4/tests/test_lxml.py +++ b/bs4/tests/test_lxml.py @@ -85,4 +85,3 @@ class LXMLXMLTreeBuilderSmokeTest(SoupTest, XMLTreeBuilderSmokeTest): @property def default_builder(self): return LXMLTreeBuilderForXML() - |