Mercurial > public > sg101

annotate gpp/legacy/html2md.py @ 290:64c188a9d31f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Adding a legacy app to contain management commands to convert the old data to the new database format. This first commit has the import_old_users command.
author Brian Neal <bgneal@gmail.com>
date Fri, 24 Dec 2010 05:28:58 +0000
parents
children 2367c4795c92
rev   line source
bgneal@290 1 """
bgneal@290 2 This module contains a class derived from Python's HTMLParser to convert HTML to
bgneal@290 3 Markdown. Currently this class only supports those HTML tags that have counter-
bgneal@290 4 parts in BBCode used by stock phpBB 2.x.
bgneal@290 5
bgneal@290 6 In other words, this class was created to help convert data from a phpBB
bgneal@290 7 forum to Markdown syntax and its scope is currently limited to that task.
bgneal@290 8
bgneal@290 9 """
bgneal@290 10 from HTMLParser import HTMLParser
bgneal@290 11 import htmlentitydefs
bgneal@290 12
bgneal@290 13
bgneal@290 14 # Let's call Markdown markup entities "elements" to avoid confusion
bgneal@290 15 # with HTML tags.
bgneal@290 16
bgneal@290 17 class ElementBase(object):
bgneal@290 18 """
bgneal@290 19 Base class for all Markdown elements.
bgneal@290 20
bgneal@290 21 """
bgneal@290 22 def __init__(self, attrs=None):
bgneal@290 23 self.data = u''
bgneal@290 24 self.attrs = dict(attrs) if attrs else {}
bgneal@290 25
bgneal@290 26 def add_data(self, data):
bgneal@290 27 self.data += data
bgneal@290 28
bgneal@290 29 def markdown(self):
bgneal@290 30 return self.data
bgneal@290 31
bgneal@290 32
bgneal@290 33 class TextElement(ElementBase):
bgneal@290 34 """
bgneal@290 35 TextElements represent text fragments not inside HTML tags.
bgneal@290 36 """
bgneal@290 37 pass
bgneal@290 38
bgneal@290 39
bgneal@290 40 class EmphasisElement(ElementBase):
bgneal@290 41 """
bgneal@290 42 An EmphasisElement is a Markdown element used to indicate emphasis and is
bgneal@290 43 represented by placing characters around text. E.g. _em_, **bold**
bgneal@290 44
bgneal@290 45 """
bgneal@290 46 def __init__(self, tag, attrs):
bgneal@290 47 super(EmphasisElement, self).__init__(attrs)
bgneal@290 48 self.tag = tag
bgneal@290 49
bgneal@290 50 def markdown(self):
bgneal@290 51 return u'%s%s%s' % (self.tag, self.data, self.tag)
bgneal@290 52
bgneal@290 53
bgneal@290 54 def create_emphasis(tag):
bgneal@290 55 """
bgneal@290 56 Returns a function that creates an EmphasisElement using the supplied
bgneal@290 57 tag.
bgneal@290 58
bgneal@290 59 """
bgneal@290 60 def inner(attrs):
bgneal@290 61 return EmphasisElement(tag, attrs)
bgneal@290 62 return inner
bgneal@290 63
bgneal@290 64
bgneal@290 65 class HtmlElement(ElementBase):
bgneal@290 66 """
bgneal@290 67 Markdown also accepts HTML markup. This element represents a HTML tag that
bgneal@290 68 maps to itself in Markdown.
bgneal@290 69
bgneal@290 70 """
bgneal@290 71 def __init__(self, tag, attrs):
bgneal@290 72 super(HtmlElement, self).__init__(attrs)
bgneal@290 73 self.tag = tag
bgneal@290 74
bgneal@290 75 def markdown(self):
bgneal@290 76 return u'<%s>%s</%s>' % (self.tag, self.data, self.tag)
bgneal@290 77
bgneal@290 78
bgneal@290 79 def create_html(tag):
bgneal@290 80 """
bgneal@290 81 Returns a function that creates a HtmlElement using the supplied tag.
bgneal@290 82
bgneal@290 83 """
bgneal@290 84 def inner(attrs):
bgneal@290 85 return HtmlElement(tag, attrs)
bgneal@290 86 return inner
bgneal@290 87
bgneal@290 88
bgneal@290 89 class QuoteElement(ElementBase):
bgneal@290 90 """
bgneal@290 91 Class to represent a blockquote in Markdown.
bgneal@290 92
bgneal@290 93 """
bgneal@290 94 def markdown(self):
bgneal@290 95 return u'> %s\n\n' % self.data.replace('\n', '\n> ')
bgneal@290 96
bgneal@290 97
bgneal@290 98 class BreakElement(ElementBase):
bgneal@290 99 """
bgneal@290 100 Class to represent a linebreak in Markdown.
bgneal@290 101
bgneal@290 102 """
bgneal@290 103 def markdown(self):
bgneal@290 104 return u' \n'
bgneal@290 105
bgneal@290 106
bgneal@290 107 class DivElement(ElementBase):
bgneal@290 108 """
bgneal@290 109 This class maps a HTML <div> into a block of text surrounded by newlines.
bgneal@290 110
bgneal@290 111 """
bgneal@290 112 def markdown(self):
bgneal@290 113 return u'\n%s\n' % self.data
bgneal@290 114
bgneal@290 115
bgneal@290 116 class LinkElement(ElementBase):
bgneal@290 117 """
bgneal@290 118 This class maps HTML <a> tags into Markdown links.
bgneal@290 119 If no data is present, the actual href is used for the link text.
bgneal@290 120
bgneal@290 121 """
bgneal@290 122 def markdown(self):
bgneal@290 123 url = self.attrs['href']
bgneal@290 124 text = self.data if self.data else url
bgneal@290 125 return u'[%s](%s)' % (text, url)
bgneal@290 126
bgneal@290 127
bgneal@290 128 class ImageElement(ElementBase):
bgneal@290 129 """
bgneal@290 130 This class maps HTML <img> tags into Markdown.
bgneal@290 131 This element assumes no alt text is present, and simply uses the word
bgneal@290 132 'image' for the alt text.
bgneal@290 133
bgneal@290 134 """
bgneal@290 135 def markdown(self):
bgneal@290 136 url = self.attrs['src']
bgneal@290 137 return u'![image](%s)' % url
bgneal@290 138
bgneal@290 139
bgneal@290 140 class CodeElement(ElementBase):
bgneal@290 141 """
bgneal@290 142 This class is used to create code blocks in Markdown.
bgneal@290 143
bgneal@290 144 """
bgneal@290 145 def markdown(self):
bgneal@290 146 return u' %s\n' % self.data.replace('\n', '\n ')
bgneal@290 147
bgneal@290 148
bgneal@290 149 # List (ordered & unordered) support:
bgneal@290 150
bgneal@290 151 class ListElement(ElementBase):
bgneal@290 152 """
bgneal@290 153 This class creates Markdown for unordered lists. The bullet() method can be
bgneal@290 154 overridden to create ordered lists.
bgneal@290 155
bgneal@290 156 """
bgneal@290 157 def __init__(self, attrs=None):
bgneal@290 158 super(ListElement, self).__init__(attrs)
bgneal@290 159 self.items = []
bgneal@290 160 self.list_nesting = 1
bgneal@290 161
bgneal@290 162 def add_data(self, data):
bgneal@290 163 self.items.append(data)
bgneal@290 164
bgneal@290 165 def bullet(self):
bgneal@290 166 return u'*'
bgneal@290 167
bgneal@290 168 def markdown(self):
bgneal@290 169 bullet_str = self.bullet()
bgneal@290 170 indent = u' ' * (4 * (self.list_nesting - 1))
bgneal@290 171 s = u''
bgneal@290 172 for item in self.items:
bgneal@290 173 s += u'\n%s%s %s' % (indent, bullet_str, item)
bgneal@290 174 return s
bgneal@290 175
bgneal@290 176
bgneal@290 177 class OrderedListElement(ListElement):
bgneal@290 178 """
bgneal@290 179 This class creates Markdown for ordered lists.
bgneal@290 180
bgneal@290 181 """
bgneal@290 182 def bullet(self):
bgneal@290 183 return '1.'
bgneal@290 184
bgneal@290 185
bgneal@290 186 class ItemElement(ElementBase):
bgneal@290 187 """
bgneal@290 188 This element is used to represent ordered & unordered list items.
bgneal@290 189
bgneal@290 190 """
bgneal@290 191 pass
bgneal@290 192
bgneal@290 193 ###############################################################################
bgneal@290 194 ###############################################################################
bgneal@290 195
bgneal@290 196 class MarkdownWriter(HTMLParser):
bgneal@290 197 """
bgneal@290 198 This class is an HTMLParser that converts a subset of HTML to Markdown.
bgneal@290 199
bgneal@290 200 """
bgneal@290 201
bgneal@290 202 elem_factories = {
bgneal@290 203 'a': LinkElement,
bgneal@290 204 'blockquote': QuoteElement,
bgneal@290 205 'br': BreakElement,
bgneal@290 206 'div': DivElement,
bgneal@290 207 'em': create_emphasis('_'),
bgneal@290 208 'img': ImageElement,
bgneal@290 209 'li': ItemElement,
bgneal@290 210 'ol': OrderedListElement,
bgneal@290 211 'pre': CodeElement,
bgneal@290 212 's': create_html('strike'),
bgneal@290 213 'strong': create_emphasis('**'),
bgneal@290 214 'u': create_html('u'),
bgneal@290 215 'ul': ListElement,
bgneal@290 216 }
bgneal@290 217
bgneal@290 218 def __init__(self):
bgneal@290 219 HTMLParser.__init__(self)
bgneal@290 220 self.reset()
bgneal@290 221
bgneal@290 222 def handle_starttag(self, tag, attrs):
bgneal@290 223 if tag in self.elem_factories:
bgneal@290 224 factory = self.elem_factories[tag]
bgneal@290 225 element = factory(attrs)
bgneal@290 226 else:
bgneal@290 227 element = TextElement()
bgneal@290 228
bgneal@290 229 self._push_elem(element)
bgneal@290 230
bgneal@290 231 def handle_endtag(self, tag):
bgneal@290 232 self._pop_elem()
bgneal@290 233
bgneal@290 234 def handle_data(self, data):
bgneal@290 235 if len(self.elem_stack) == 0:
bgneal@290 236 self._push_elem(TextElement())
bgneal@290 237 self._add_data(data)
bgneal@290 238
bgneal@290 239 def handle_entityref(self, name):
bgneal@290 240 try:
bgneal@290 241 text = unichr(htmlentitydefs.name2codepoint[name])
bgneal@290 242 except KeyError:
bgneal@290 243 text = name
bgneal@290 244 self.handle_data(text)
bgneal@290 245
bgneal@290 246 def handle_charref(self, name):
bgneal@290 247 self.handle_data(unichr(int(name)))
bgneal@290 248
bgneal@290 249 def reset(self):
bgneal@290 250 HTMLParser.reset(self)
bgneal@290 251 self.elem_stack = []
bgneal@290 252 self.elements = []
bgneal@290 253 self.list_nesting = 0
bgneal@290 254
bgneal@290 255 def _push_elem(self, tag):
bgneal@290 256 if len(self.elem_stack) and isinstance(self.elem_stack[-1], TextElement):
bgneal@290 257 self._pop_elem()
bgneal@290 258 if isinstance(tag, ListElement):
bgneal@290 259 self.list_nesting += 1
bgneal@290 260 tag.list_nesting = self.list_nesting
bgneal@290 261 self.elem_stack.append(tag)
bgneal@290 262
bgneal@290 263 def _pop_elem(self):
bgneal@290 264 element = self.elem_stack.pop()
bgneal@290 265 if isinstance(element, ListElement):
bgneal@290 266 self.list_nesting -= 1
bgneal@290 267 if len(self.elem_stack):
bgneal@290 268 self.elem_stack[-1].add_data(element.markdown())
bgneal@290 269 else:
bgneal@290 270 self.elements.append(element)
bgneal@290 271
bgneal@290 272 def _add_data(self, data):
bgneal@290 273 self.elem_stack[-1].add_data(data)
bgneal@290 274
bgneal@290 275 def markdown(self):
bgneal@290 276 while len(self.elem_stack):
bgneal@290 277 self._pop_elem()
bgneal@290 278 text_list = [e.markdown() for e in self.elements]
bgneal@290 279 return u''.join(text_list)