Trac Wiki to Markdown converter

trac2down.py

#!/usr/bin/python
# vim:set fileencoding=utf-8 sw=2 ai:

# downloaded from https://gist.github.com/tcchau/4628317
# ... which is a fork from https://gist.github.com/sgk/1286682

import sqlite3
import datetime
import re
# additional imports --ah
import os
import sys

SQL = '''
  select
      name, version, time, author, text
    from
      wiki w
    where
      version = (select max(version) from wiki where name = w.name)
'''

# Modified expression to generate a file for every version (overwriting previous version) --ah
SQLgit = '''
  select
      name, version, time, author, text, comment
    from
      wiki w
    order by time
'''

# exclude wiki pages starting with.... --ah
excludepages = [ "Trac" ]

# Add function to try to preserve author information in git --ah
def gitauthor(author):
  if author == "foo@bar":
    return " --author \"Foo Bar <" + author + ">\" "
  # add more here if you want to preserve author information for edits...
  elif author == "trac":
    return " --author \"trac <" + author + "@example.com>\" "
  else:
    print "WARNING: unknown author: " + author
    return ""

# helper functions from https://gist.github.com/gazpachoking/9540849

def convert_wiki_link(link):
  if link.startswith('wiki:'):
    link = link[5:]
  return link.strip("'").replace('/', '-')


def sub_full_wiki_link(m):
  return '[%s](%s)' % (m.group(2), convert_wiki_link(m.group(1)))


def sub_simple_wiki_link(m):
  return '[[%s]]' % convert_wiki_link(m.group(1))


def sub_fenced_block(m):
  if m.group(1) == 'html':
    return '\n%s\n' % m.group(2)
  elif m.group(1):
    return '```%s\n%s\n```' % (m.group(1), m.group(2))
  return '```\n%s\n```' % m.group(2)


def sub_table(m):
  lines = []
  for group in m.group(0).strip().split('\n'):
    lines.append(' | '.join(group.strip().split('||')).strip())
  width = len(m.group(1).strip().split('||')) - 2
  lines.insert(1, '| %s |' % ' | '.join('---' for x in range(width)))
  return '\n%s\n' % '\n'.join(lines)

# use alternative SQL if using git to preserve history of changes --ah
if "--git" in sys.argv:
  SQL = SQLgit

conn = sqlite3.connect('../trac.db')
result = conn.execute(SQL)
for row in result:
  name = row[0]
  version = row[1]
  time = row[2]
  author = row[3]
  text = row[4]
  try:
    comment = row[5]
  except IndexError:
    comment = "empty comment (added by migrator)"

  # add possibility to exclude certain pages. --ah

  if name.startswith(tuple(excludepages)):
    print "Skipping " + name + " (version " + str(version) + ")"
    continue

  text = re.sub('\r\n', '\n', text)
  text = re.sub(r'{{{(.*?)}}}', r'`\1`', text)
  # below 2 line from https://gist.github.com/gazpachoking/9540849
  text = re.sub(r'(?sm){{{\n(?:#!([a-z]+)\n)?(.*?)\n}}}', sub_fenced_block, text)
  text = re.sub(r'(?m)^(\|\|[^\n]+\|\| *\n?)+$', sub_table, text)

  def indent4(m):
    return '\n    ' + m.group(1).replace('\n', '\n    ')
  text = re.sub(r'(?sm){{{\n(.*?)\n}}}', indent4, text)
  # Modified lines below to allow sloppy trailing spaces --ah
  #text = re.sub(r'(?m)^====\s+(.*?)\s+====$', r'#### \1', text)
  text = re.sub(r'(?m)^====\s+(.*?)\s+====\s*$', r'#### \1', text)
  #text = re.sub(r'(?m)^===\s+(.*?)\s+===$', r'### \1', text)
  text = re.sub(r'(?m)^===\s+(.*?)\s+===\s*$', r'### \1', text)
  #text = re.sub(r'(?m)^==\s+(.*?)\s+==$', r'## \1', text)
  text = re.sub(r'(?m)^==\s+(.*?)\s+==\s*$', r'## \1', text)
  #text = re.sub(r'(?m)^=\s+(.*?)\s+=$', r'# \1', text)
  text = re.sub(r'(?m)^=\s+(.*?)\s+=\s*$', r'# \1', text)
  text = re.sub(r'^       * ', r'****', text)
  text = re.sub(r'^     * ', r'***', text)
  text = re.sub(r'^   * ', r'**', text)
  text = re.sub(r'^ * ', r'*', text)
  text = re.sub(r'^ \d+. ', r'1.', text)

  a = []
  for line in text.split('\n'):
    if not line.startswith('    '):
      #line = re.sub(r'\[(https?://[^\s\[\]]+)\s([^\[\]]+)\]', r'[\2](\1)', line)
      #line = re.sub(r'\[(wiki:[^\s\[\]]+)\s([^\[\]]+)\]', r'[\2](/\1/)', line)
      #line = re.sub(r'\!(([A-Z][a-z0-9]+){2,})', r'\1', line)
      #line = re.sub(r'\'\'\'(.*?)\'\'\'', r'*\1*', line)
      #line = re.sub(r'\'\'(.*?)\'\'', r'_\1_', line)
      # --- above original replaced with parts from https://gist.github.com/gazpachoking/9540849
      line = re.sub(r'(?<!\[)\[([^\s\[\]]+?)\]', sub_simple_wiki_link, line)
      line = re.sub(r'\[([a-z]+?://[^\s\[\]]+)\s([^\[\]]+)\]', r'[\2](\1)', line)
      line = re.sub(r'\[(wiki:?[^\s\[\]]+)\s([^\[\]]+)\]', sub_full_wiki_link, line)
      line = re.sub(r'\!(([A-Z][a-z0-9]+){2,})', r'\1', line)
      line = re.sub(r'\'\'\'(.*?)\'\'\'', r'*\1*', line)
      line = re.sub(r'\'\'(.*?)\'\'', r'_\1_', line)
    a.append(line)
  text = '\n'.join(a)

  # Modification to create directory path when needed. --ah
  if "/" in name:
    try:
      os.makedirs(os.path.dirname(name))
    except:
      pass


  fp = file('%s.md' % name, 'w')
  fp.write(text.encode('utf-8'))
  fp.close()

  # commit newly created/overwritten file to preserve history. --ah
  if comment != "":
   comment = "\n\n%s" %comment
  if "--git" in sys.argv:
    os.system("git add .; git commit --quiet " + gitauthor(author) + " --date=\"" + datetime.datetime.fromtimestamp(time/1000000).strftime('%Y/%m/%d %H:%M:%S') + "\" -avm \"Import " + name + ", version " + str(version) + str(comment) + "\"")