Making sgmlib more liberal

[Jeff Bowden]

I've written a simple class derived from sgmllib.SGMLParser to extract
text from html pages. So far it's worked pretty well except for a few
cases where I get exceptions. I've managed to work around these
problems by overriding parse_declaration.

Since parse_declaration is preceded by the comment

# Internal -- parse declaration (for use by subclasses).

I am thinking my workaround might possibly stop working with future
versions of sgmllib so I'm looking for a more correct alternative.
Any suggestions?

Here's my code:

_endTag = re.compile(r'>')

class SGML2TextParser(sgmllib.SGMLParser):
def __init__(self, f, ignoretags=['script']):
sgmllib.SGMLParser.__init__(self)
self.f = f
self.ignoretags = ignoretags
self.tag = ''

def handle_starttag(self, tag, attrs):
self.tag = tag

def handle_data(self, data):
if self.tag not in self.ignoretags:
self.f.write(data)

def handle_charref(self, name):
try:
n = int(name)
self.handle_data(unichr(n))
except ValueError:
pass

# DANGER: overriding internal function
def parse_declaration(self, i):
try:
return sgmllib.SGMLParser.parse_declaration(self, i)
except:
match = _endTag.search(self.rawdata, i)
return match and match.end(0) or -1

def extractText(html_text):
s = StringIO.StringIO()
x = SGML2TextParser(s)
x.feed(html_text)
return s.getvalue()