Mw2html

From Noah.org
Revision as of 14:17, 25 August 2008 by Root (Talk | contribs)

(diff) ← Older revision | Latest revision (diff) | Newer revision → (diff)
Jump to: navigation, search

This was originally from here. It's not as useful to me know, but I keep it around because there isn't much else I've found that will do the same thing.

#!/usr/bin/env python
"""mw2html - Mediawiki to static HTML

I use this to create a personal website from a local mediawiki
installation.  No search functionality.  Hacks the Monobook skin and
the produced HTML.

Connelly Barnes 2005.  Public domain.
"""

__version__ = '0.1.0.0'

import re
import sys
import getopt
import random
import urllib
import urllib2
import textwrap
import urlparse
import os, os.path
import errno
import sha

try:
  set
except:
  from sets import Set as set

try:
  import htmldata
except:
  print 'Requires Python htmldata module:'
  print '  http://oregonstate.edu/~barnesc/htmldata/'
  sys.exit()

MOVE_HREF          = 'movehref'
MADE_BY_COMMENT    = '<!-- Content generated by Mediawiki and mw2html -->'
INDEX_HTML = 'index.html'
url_filename_cache = {}
wrote_file_set     = set()

MONOBOOK_SKIN      = 'monobook'    # Constant identifier for Monobook.

class Config:
  """
  Instances contain all options passed at the command line.
  """
  def __init__(self, rooturl, outdir,
               flatten=True, lower=True, index=None, clean=True,
               sidebar=None, hack_skin=True,
               made_by=True, overwrite=False, footer=None,
               skin=MONOBOOK_SKIN, move_href=True,
               remove_png=True, remove_history=True):
    self.rooturl         = rooturl
    self.outdir          = os.path.abspath(outdir)
    self.flatten         = flatten
    self.lower           = lower
    self.index           = index
    self.clean           = clean
    self.sidebar         = sidebar
    self.hack_skin       = hack_skin
    self.made_by         = made_by
    self.overwrite       = overwrite
    self.footer          = footer
    self.skin            = skin
    self.move_href       = move_href
    if self.sidebar is not None:
      self.sidebar       = os.path.abspath(self.sidebar)
    if self.footer is not None:
      self.footer        = os.path.abspath(self.footer)
    self.remove_png      = remove_png
    self.remove_history  = remove_history


def post_filename_transform(filename, config):
  """
  User-customizable filename transform.

  Here filename is the full filename in the output directory.
  Returns modified full filename.
  """
  return filename


def monobook_fix_html_sidebar(doc, config):
  """
  Sets sidebar for Mediawiki 1.4beta6 Monobook HTML output.
  """
  if config.made_by:
    doc = doc.replace('<html xmlns=', MADE_BY_COMMENT + '\n<html xmlns=')

  SIDEBAR_ID = 'SIDEBAR' + sha.new(str(random.random())).hexdigest()

  # Remove sidebar HTML
  doc = re.sub(
    r'(<!-- end content -->)[\s\S]+?' +
    r'(<!-- end of the left \(by default at least\) column -->)',
    r'\1<div class="visualClear"></div></div></div></div>' + SIDEBAR_ID + r'\2', doc)

  pre_sidebar = """
    <div id="column-one">
      <div id="p-cactions" class="portlet"></div>
      <div class="portlet" id="p-personal"></div>
      <div class="portlet" id="p-logo"></div>
      <div class="portlet" id="p-nav">
  """

  post_sidebar = """
      </div>
      <div id="p-search" class="portlet"></div>
      <div class="portlet" id="p-tb"></div>
    </div>
    <!-- end left column -->
  """

  sidebar_content = ''
  if config.sidebar != None:
    f = open(config.sidebar, 'rU')
    sidebar_content = f.read()
    f.close()

  sidebar_content = pre_sidebar + sidebar_content + post_sidebar

  doc = doc.replace(SIDEBAR_ID, sidebar_content)

  doc = re.sub(
    r'<div id="f-poweredbyico">[\s\S]+?(<ul id="f-list">)',
    r'\1', doc)

  # Remove edit links
  doc = re.sub(r'<div class="editsection"[\s\S]+?</div>', r'', doc)

  # Remove page has been accessed X times list item.
  doc = re.sub(r'<li id="f-viewcount">[\s\S]+?</li>', r'', doc)

  # Remove disclaimers list item.
  doc = re.sub(r'<li id="f-disclaimer">[\s\S]+?</li>', r'', doc)

  # Replace remaining text with footer, if available.
  if config.footer is not None:
    s1 = '<div id="footer">'
    s2 = '</div>'
    i1 = doc.index(s1)
    i2 = doc.index(s2, i1)
    f = open(config.footer, 'rU')
    footer_text = f.read()
    f.close()
    doc = doc[:i1+len(s1)] + footer_text + doc[i2:]

  return doc


def fix_move_href_tags(doc, config):
  """
  Return copy of doc with all MOVE_HREF tags removed.
  """
  while '<' + MOVE_HREF in doc:
    i1 = doc.index('<' + MOVE_HREF)
    i2 = doc.index('</' + MOVE_HREF, i1+1)
    i3 = doc.index('>', i2+1)
    (start, end) = (i1, i3+1)
    tags = htmldata.tagextract(doc[start:end])
    assert tags[0][0] == MOVE_HREF
    assert tags[-1][0] == '/' + MOVE_HREF
    href = tags[0][1].get('href', '')
    new_tags = []
    for tag in tags[1:-1]:
      if len(tag) == 2:
        if 'href' in tag[1]:
          if href == '':
            continue
          tag[1]['href'] = href
      new_tags += [tag]
    doc = doc[:start] + htmldata.tagjoin(new_tags) + doc[end:]
  return doc


def html_remove_image_history(doc, config):
  """
  Remove image history and links to information.
  """
  doc = re.sub(r'<h2>Image history</h2>[\s\S]+?</ul>', r'', doc)
  doc = re.sub(r'<h2>Image links</h2>[\s\S]+?</ul>', r'', doc)
  return doc


def post_html_transform(doc, url, config):
  """
  User-customizable HTML transform.

  Given an HTML document (with URLs already rewritten), returns
  modified HTML document.
  """
  if config.hack_skin:
    if config.skin == MONOBOOK_SKIN:
      doc = monobook_fix_html_sidebar(doc, config)
      doc = monobook_hack_skin_html(doc, config)
    else:
      raise ValueError('unknown skin')
  if config.move_href:
    doc = fix_move_href_tags(doc, config)
  if config.remove_history:
    doc = html_remove_image_history(doc, config)
  return doc


def monobook_hack_skin_html(doc, config):
  """
  Hacks Monobook HTML output: use CSS ids for hacked skin.

  See monobook_hack_skin_css.
  """
  doc = doc.replace('<div id="globalWrapper">', '<div id="globalWrapperHacked">')
  doc = doc.replace('<div id="footer">', '<div id="footerHacked">')
  doc = doc.replace('</body>', '<br></body>')
  return doc


def monobook_hack_skin_css(doc, url, config):
  """
  Hacks Mediawiki 1.4beta6 Monobook main CSS file for better looks.

  Removes flower background.  Defines *Hacked CSS ids, so we can add
  an orange bar at the top, and clear the orange bar right above the
  footer.
  """
  if not url.endswith('monobook/main.css'):
    return doc

  doc = "/* Monobook skin automatically modified by mw2html. */" + doc
  doc = doc.replace('url("headbg.jpg")', '')

  doc += """
    /* Begin hacks by mw2html */

    #globalWrapperHacked {
      font-size:127%;
      width: 100%;
      background-color: White;
      border-top: 1px solid #fabd23;
      border-bottom: 1px solid #fabd23;
      margin: 0.6em 0em 1em 0em;
      padding: 0em 0em 1.2em 0em;
    }

    #footerHacked {
      background-color: White;
      margin: 0.6em 0em 0em 0em;
      padding: 0.4em 0em 0em 0em;
      text-align: center;
      font-size: 90%;
    }

    #footerHacked li {
      display: inline;
      margin: 0 1.3em;
    }
    """

  c1 = '#column-one { padding-top: 160px; }'
  c2 = '#column-one { padding-top: 3.0em; }'
  assert c1 in doc

  doc = doc.replace(c1, '/* edit by mw2html */\n' + c2 +
                        '\n/* end edit by mw2html */\n')

  # Remove external link icons.
  if config.remove_png:
    doc = re.sub(r'#bodyContent a\[href \^="http://"\][\s\S]+?\}', r'', doc)

  return doc


def post_css_transform(doc, url, config):
  """
  User-customizable CSS transform.

  Given a CSS document (with URLs already rewritten), returns
  modified CSS document.
  """
  if config.hack_skin:
    if config.skin == MONOBOOK_SKIN:
      doc = monobook_hack_skin_css(doc, url, config)
    else:
      raise ValueError('unknown skin')
  return doc


def url_to_filename(url, config):
  """
  Translate a full url to a full filename (in local OS format) under outdir.
  """
  url = split_section(url)[0]
  if url in url_filename_cache:
    return url_filename_cache[url]

  part = url
  if part.lower().startswith('http://'):
    part = part[len('http://'):]
  L = part.strip('/').split('/')
  L = [urllib.quote_plus(x) for x in L]
  if len(L) <= 1 or not '.' in L[-1]:
    # url ends with a directory name.  Store it under index.html.
    L += [INDEX_HTML]

  # Local filename relative to outdir
  # (More transformations are made to this below...).
  subfile = os.sep.join(L)

  # Fix up extension based on mime type.
  fix_ext = True
  try:
    f = urllib2.urlopen(url)
  except urllib2.URLError, e:
    fix_ext = False

  if fix_ext:
    mimetype = f.info().type.lower().split(' ')[0]

    # Maps mimetype to file extension
    MIME_MAP = {
     'image/jpeg': 'jpg', 'image/png': 'png', 'image/gif': 'gif',
     'image/tiff': 'tiff', 'text/plain': 'txt', 'text/html': 'html',
     'text/rtf': 'rtf', 'text/css': 'css', 'text/sgml': 'sgml',
     'text/xml': 'xml', 'application/zip': 'zip'
    }

    if mimetype in MIME_MAP:
      (root, ext) = os.path.splitext(subfile)
      ext = '.' + MIME_MAP[mimetype]
      subfile = root + ext

  if config.lower:
    subfile = subfile.lower()

  ans = os.path.join(config.outdir, subfile)

  if config.flatten:
    ans = flatten_filename(url, config, ans)

  if config.clean:
    ans = clean_filename(url, config, ans)

  if config.index != None:
    ans = move_to_index_if_needed(config, ans)

  ans = find_unused_filename(ans, file_exists_in_written_set)

  ans = post_filename_transform(ans, config)

  # Cache and return answer.
  wrote_file_set.add(os.path.normcase(os.path.normpath(ans)))
  url_filename_cache[url] = ans
  return ans


def file_exists_in_written_set(filename):
  return os.path.normcase(os.path.normpath(filename)) in wrote_file_set


def find_unused_filename(filename, exists=os.path.exists):
  """
  Return 'file' if 'file' doesn't exist, otherwise 'file1', 'file2', etc.

  Existance is determined by the callable exists(), which takes
  a filename and returns a boolean.
  """
  if not exists(filename):
    return filename
  (head, tail) = os.path.split(filename)
  i = 1
  while True:
    numbered = (os.path.splitext(tail)[0] + str(i) +
                os.path.splitext(tail)[1])
    fullname = os.path.join(head, numbered)
    if not exists(fullname):
      return fullname
    i += 1


def clean_filename(url, config, ans):
  # Split outdir and our file/dir under outdir
  # (Note: ans may not be a valid filename)
  (par, ans) = (ans[:len(config.outdir)], ans[len(config.outdir):])
  if ans.startswith(os.sep):
    ans = ans[1:]

  # Replace % escape codes with underscores, dashes with underscores.
  while '%%' in ans:
    ans = ans[:ans.index('%%')] + '_' + ans[ans.index('%%')+2:]
  while '%25' in ans:
    ans = ans[:ans.index('%25')] + '_' + ans[ans.index('%25')+5:]
  while '%' in ans:
    ans = ans[:ans.index('%')] + '_' + ans[ans.index('%')+3:]
  ans = ans.replace('-', '_')
  while '__' in ans:
    ans = ans.replace('__', '_')
  while '_.' in ans:
    ans = ans.replace('_.', '.')

  # Rename math thumbnails
  if '/math/' in url:
    tail = os.path.split(ans)[1]
    if os.path.splitext(tail)[1] == '.png':
      tail = os.path.splitext(tail)[0]
      if set(tail) <= set('0123456789abcdef') and len(tail) == 32:
        ans = 'math_' + sha.new(tail).hexdigest()[:4] + '.png'
  return os.path.join(par, ans)


def move_to_index_if_needed(config, ans):
  if ans.endswith(config.index):
    ans = ans[:len(ans)-len(config.index)] + INDEX_HTML
  return ans


def flatten_filename(url, config, filename):
  def get_fullname(relname):
    return os.path.join(config.outdir, relname)

  orig_ext = os.path.splitext(filename)[1]
  (head, tail) = os.path.split(filename)
  if tail == INDEX_HTML:
    (head, tail) = os.path.split(head)
  ans = tail
  if os.path.splitext(ans)[1] != orig_ext:
    ans = os.path.splitext(ans)[0] + orig_ext
  return os.path.join(config.outdir, ans)


def split_section(url):
  """
  Splits into (head, tail), where head contains no '#' and is max length.
  """
  if '#' in url:
    i = url.index('#')
    return (url[:i], url[i:])
  else:
    return (url, '')


def rewrite_external_url(url, config):
  """
  Rewrite any URL that could not be stored locally.

  To not rewrite any external URLs, simply return url.
  """
  # If could not be stored locally, but in same domain, return ''.
  if get_domain(url) == get_domain(config.rooturl):
    return ''
  return url


def url_to_relative(url, cururl, config):
  """
  Translate a full url to a filename (in URL format) relative to cururl.
  """
  cururl = split_section(cururl)[0]
  (url, section) = split_section(url)

  L1 = url_to_filename(url,    config).replace(os.sep, '/').split('/')
  L2 = url_to_filename(cururl, config).replace(os.sep, '/').split('/')

  while L1 != [] and L2 != [] and L1[0] == L2[0]:
    L1 = L1[1:]
    L2 = L2[1:]

  return urllib.quote('../' * (len(L2) - 1) + '/'.join(L1)) + section


def parse_css(doc, url, config):
  """
  Returns (modified_doc, new_urls), where new_urls are absolute URLs for
  all links found in the CSS.
  """
  new_urls = []  

  L = htmldata.urlextract(doc, url, 'text/css')
  for item in L:
    # Store url locally.
    u = item.url
    new_urls += [u]
    item.url = url_to_relative(u, url, config)

  newdoc = htmldata.urljoin(doc, L)
  newdoc = post_css_transform(newdoc, url, config)

  return (newdoc, new_urls)


def get_domain(u):
  """
  Get domain of URL.
  """
  ans = urlparse.urlparse(u)[1]
  if ':' in ans:
    ans = ans[:ans.index(':')]
  return ans


def should_follow(rooturl, url):
  """
  Returns boolean for whether url should be spidered.

  Given that 'url' was linked to from site 'rooturl', return whether
  'url' should be spidered as well.
  """
  # False if different domains.
  if get_domain(rooturl) != get_domain(url):
    return False

  # False if multiple query fields.
  if url.count('&') >= 1:
    return False

  if 'MediaWiki:' in url or 'Special:' in url:
    return False

  return True


def parse_html(doc, url, config):
  """
  Returns (modified_doc, new_urls), where new_urls are absolute URLs for
  all links we want to spider in the HTML.
  """
  BEGIN_COMMENT_REPLACE = '<BEGINCOMMENT-' + str(random.random()) + '>'
  END_COMMENT_REPLACE   = '<ENDCOMMENT-' + str(random.random()) + '>'

  new_urls = []  

  # Temporarily "get rid" of comments so htmldata will find the URLs
  # in the funky "<!--[if" HTML hackery for IE.
  doc = doc.replace('<!--', BEGIN_COMMENT_REPLACE)
  doc = doc.replace('-->', END_COMMENT_REPLACE)

  L = htmldata.urlextract(doc, url, 'text/html')
  for item in L:
    u = item.url
    if should_follow(url, u):
      # Store url locally.
      new_urls += [u]
      item.url = url_to_relative(u, url, config)
    else:
      item.url = rewrite_external_url(item.url, config)

  newdoc = htmldata.urljoin(doc, L)
  newdoc = newdoc.replace(BEGIN_COMMENT_REPLACE, '<!--')
  newdoc = newdoc.replace(END_COMMENT_REPLACE, '-->')
  newdoc = post_html_transform(newdoc, url, config)

  return (newdoc, new_urls)
  

def run(config, out=sys.stdout):
  """
  Code interface.
  """
  if urlparse.urlparse(config.rooturl)[1].lower().endswith('wikipedia.org'):
    out.write('Please do not use robots with the Wikipedia site.\n')
    out.write('Instead, install the Wikipedia database locally and use mw2html on\n')
    out.write('your local installation.  See the Mediawiki site for more information.\n')
    sys.exit(1)

  # Number of files saved
  n = 0

  if not config.overwrite and os.path.exists(config.outdir):
    out.write('Error: Directory exists: ' + str(config.outdir) )
    sys.exit(1)

  complete = set()
  pending  = set([config.rooturl])

  while len(pending) > 0:
    url      = pending.pop()
    if url in complete:
      continue
    complete.add(url)
    try:
      f        = urllib2.urlopen(url)
    except urllib2.URLError, e:
      try:
        out.write(str(e.code) + ': ' + url + '\n\n')
      except:
        out.write('Error opening: ' + url + '\n\n')
      continue
    doc      = f.read()
    mimetype = f.info().type.lower().split(' ')[0]
    f.close()

    new_urls = []

    if mimetype == 'text/html':
      (doc, new_urls) = parse_html(doc, url, config)
    elif mimetype == 'text/css':
      (doc, new_urls) = parse_css(doc, url, config)

    # Enqueue URLs that we haven't yet spidered.
    for u in new_urls:
      if u not in complete:
        # Strip off any #section link.
        if '#' in u:
          u = u[:u.index('#')]
        pending.add(u)

    mode = ['wb', 'w'][mimetype.startswith('text')]

    # Save modified content to disk.
    filename = url_to_filename(url, config)

    # Make parent directory if it doesn't exist.
    try:
      os.makedirs(os.path.split(filename)[0])
    except OSError, e:
      if e.errno != errno.EEXIST:
        raise

    # Not really needed since we checked that the directory
    # outdir didn't exist at the top of run(), but let's double check.
    if os.path.exists(filename) and not config.overwrite:
      out.write('File already exists: ' + str(filename))
      sys.exit(1)

    f = open(filename, mode)
    f.write(doc)
    f.close()

    out.write(url + '\n => ' + filename + '\n\n')
    n += 1

  out.write(str(n) + ' file(s) saved\n')


def usage():
  """
  Print command line options.
  """
  usage_str = """
  mw2html url outdir [options]

  Converts an entire Mediawiki site into static HTML.
  Tested only with Mediawiki 1.4beta6 Monobook output.
  WARNING: This is a recursive robot that ignores robots.txt.  Use with care.

    url                  - URL of mediawiki page to convert to static HTML.
    outdir               - Output directory.

    -f, --force          - Overwrite existing files in outdir.
    --no-flatten         - Do not flatten directory structure.
    --no-lower           - Retain original case for output filenames and dirs.
    --no-clean           - Do not clean up filenames (clean replaces
                           non-alphanumeric chars with _, renames math thumbs).
    --no-hack-skin       - Do not modify skin CSS and HTML for looks.
    --no-made-by         - Suppress "generated by" comment in HTML source.
    --no-move-href       - Disable <movehref> tag. [1]
    --no-remove-png      - Retain external link PNG icons.
    --no-remove-history  - Retain image history and links to information.
    -l, --left=a.html    - Paste HTML fragment file into left sidebar.
    -t, --top=a.html     - Paste HTML fragment file into top horiz bar.
    -b, --bottom=a.html  - Paste HTML fragment file into footer horiz bar.
    -i, --index=filename - Move given filename in outdir to index.html.

  Example Usage:
    mw2html http://127.0.0.1/mywiki/ out -f -i main_page.html -l sidebar.html

    Freezes wiki into 'out' directory, moves main_page.html => index.html,
    assumes sidebar.html is defined in the current directory.

  [1]. The <movehref> tag.
       Wiki syntax: <html><movehref href="a"></html>...<html></movehref></html>.
       When enabled, this tag will cause all href= attributes inside of it to be
       set to the given location.  This is useful for linking images.
       In MediaWiki, for the <html> tag to work, one needs to enable $wgRawHtml
       and $wgWhitelistEdit in LocalSettings.php.  A <movehref> tag with no href
       field will remove all links inside it.

  """

  print textwrap.dedent(usage_str.strip('\n'))
  sys.exit(1)


def main():
  """
  Command line interface.
  """
  try:
    (opts, args) = getopt.gnu_getopt(sys.argv[1:], 'fl:t:b:i:',
                   ['force', 'no-flatten', 'no-lower', 'no-clean',
                    'no-hack-skin', 'no-made-by', 'left=',
                    'top=', 'bottom=', 'index=', 'no-move-href',
                    'no-remove-png', 'no-remove-history'])
  except getopt.GetoptError:
    usage()

  # Parse non-option arguments
  try:
    (rooturl, outdir) = args
  except ValueError:
    usage()
  config = Config(rooturl=rooturl, outdir=outdir)

  # Parse option arguments
  for (opt, arg) in opts:
    if opt in ['-f', '--force']:
      config.overwrite      = True
    if opt in ['--no-flatten']:
      config.flatten        = False
    if opt in ['--no-lower']:
      config.lower          = False
    if opt in ['--no-clean']:
      config.clean          = False
    if opt in ['--no-hack-skin']:
      config.hack_skin      = False
    if opt in ['--no-made-by']:
      config.made_by        = False
    if opt in ['--no-move-href']:
      config.move_href      = False
    if opt in ['--no-remove-png']:
      config.remove_png     = False
    if opt in ['--no-remove-history']:
      config.remove_history = False
    if opt in ['-l', '--left']:
      config.sidebar        = os.path.abspath(arg)
    if opt in ['-t', '--top']:
      raise NotImplementedError
      config.header         = os.path.abspath(arg)
    if opt in ['-b', '--bottom']:
      config.footer         = os.path.abspath(arg)
    if opt in ['-i', '--index']:
      config.index          = arg

  # Run program
  run(config)


if __name__ == '__main__':
  main()