# HG changeset patch # User Oben Sonne # Date 1284150886 -7200 # Fri Sep 10 22:34:46 2010 +0200 # Node ID a326d2c4a1bbf6b15fd84ba8adc72553b07fb0c2 # Parent 01b0235001f6fdda23a34b95a93cc5d371d160d4 Integrate math expressions into DB. This has the advantages that a writeable math image location is not required anymore, unused math images easily can be removed and still used images cannot accidentally be removed. diff --git a/README b/README --- a/README +++ b/README @@ -52,15 +52,13 @@ Structure of a local wiki ~~~~~~~~~~~~~~~~~~~~~~~~~ -A new wiki at ``/path/to/wiki`` consists of the following files and directories: +A new wiki at ``/path/to/wiki`` consists of the following files and directories: ``wiki.db`` An `SQLite`_ database containing wiki pages. ``media/images`` - Wiki image links refer to files located here (see in-wiki help for details). -``media/math`` - Used to store rendered formula images (see in-wiki help for details). - + Wiki image links refer to files located here (see in-wiki help for details). + The whole wiki s file based which makes it easy to share it across multiple computers, e.g. using services like `UbuntuOne`_ or `Dropbox`_. @@ -89,13 +87,10 @@ ``DIGGIE_WIKINAME = "Diggie"`` Name of the wiki. - - ``DIGGIE_MEDIA = ""`` - Path relative to ``MEDIA_ROOT`` (and ``MEDIA_URL``) to use for wiki - content files (e.g. images). Must use *slashes*! The server needs write - access to ``MEDIA_ROOT/DIGGIE_MEDIA/math`` for saving rendered math - images. - + + ``DIGGIE_IMAGES = ""`` + Path (relative to ``MEDIA_URL``) where wiki images are located. + ``DIGGIE_MDX = []`` List of additional or custom markdown extensions. @@ -135,7 +130,7 @@ Personally I use it in Google Chrome 6. If you find something not working in other browsers, feel free to fix this and share your patches by forking the source repository. I'll happily integrate improvements, except dirty hacks for -old IE versions. +old IE versions. =============================================================================== Feature wish list @@ -145,6 +140,9 @@ match). - Compress page history. - Provide a *diff* view for page revisions. +- Support uploading of files and images. This is not needed when using + *Diggie* as a local wiki (where you use your file browser to put files into + a wiki) but when the *Diggie* app runs on a remote machine. =============================================================================== Changes diff --git a/app/diggie/cmdline.py b/app/diggie/cmdline.py --- a/app/diggie/cmdline.py +++ b/app/diggie/cmdline.py @@ -163,7 +163,6 @@ _die("path already exists") os.makedirs(join(opts.path, "media", "images")) - os.makedirs(join(opts.path, "media", "math")) LOCALSITE['settings.py'] = LOCALSITE['settings.py'] % opts.name dizzie = join(opts.path, "dizzie") diff --git a/app/diggie/fields.py b/app/diggie/fields.py deleted file mode 100644 --- a/app/diggie/fields.py +++ /dev/null @@ -1,38 +0,0 @@ -import markdown - -from django.db.models import TextField - -from diggie.mdx import DiggieExtension -from diggie.settings import MDX - -class MarkdownTextField (TextField): - """ - A TextField that automatically implements DB-cached Markdown translation. - - NOTE: The MarkdownTextField is not able to check whether the model - defines any other fields with the same name as the HTML field it - attempts to add - if there are other fields with this name, a - database duplicate column error will be raised. - - Based on http://djangosnippets.org/snippets/882/ - - """ - def contribute_to_class (self, cls, name): - TextField(editable=False).contribute_to_class(cls, "html") - TextField(editable=False).contribute_to_class(cls, "targets") - super(MarkdownTextField, self).contribute_to_class(cls, name) - - def pre_save (self, model, add): - value = getattr(model, self.attname) - - dx = DiggieExtension(None) - mx = ["abbr", "toc", "tables", "def_list", "footnotes", dx] + MDX - md = markdown.Markdown(extensions=mx, output_format="xhtml1") - html = md.convert(value) - - setattr(model, "html", html) - setattr(model, "targets", ",".join(md.targets)) - return value - - def __unicode__ (self): - return self.attname diff --git a/app/diggie/math.py b/app/diggie/math.py deleted file mode 100644 --- a/app/diggie/math.py +++ /dev/null @@ -1,63 +0,0 @@ -from subprocess import Popen, PIPE, STDOUT -import tempfile -from os.path import join -import re -from shutil import rmtree - -_LATEX = r""" -\documentclass{article} -\usepackage{color} -\usepackage{amsmath} -\usepackage{amsfonts} -\usepackage[active,tightpage]{preview} -\definecolor{bg}{rgb}{ -1,1,1 -} -\definecolor{fg}{rgb}{ -0,0,0 -} -\pagestyle{empty} -\pagecolor{bg} -\begin{document} -\color{fg} -\begin{preview} -$ -%s -$ -\end{preview} -\end{document} -""" - -class MathException(Exception): - - def __init__(self, msg, output): - super(MathException, self).__init__(msg) - self.output = output - -def math_to_png(source, dpi, fname): - - def run(cmd): - try: - p = Popen(cmd, stdout=PIPE, stderr=STDOUT, cwd=tdir) - out = p.communicate()[0] - except OSError, e: - rmtree(tdir) - raise MathException("failed to run *%s*" % cmd[0], e) - if p.returncode != 0: - rmtree(tdir) - raise MathException("*%s* had problems" % cmd[0], out) - return out - - tdir = tempfile.mkdtemp() - with open(join(tdir, "math.latex"), 'w') as fp: - fp.write(_LATEX % source) - cmd = ["latex", "-interaction=nonstopmode", "math.latex"] - run(cmd) - cmd = ["dvipng", "-D", dpi, "-z", "9", "--depth*", "-o", fname, - "math.dvi"] - output = run(cmd) - rmtree(tdir) - depth = (re.findall(r'depth=([0-9]+)', output) + [0])[0] - valign = -int(depth) - return valign - diff --git a/app/diggie/mdx.py b/app/diggie/mdx.py --- a/app/diggie/mdx.py +++ b/app/diggie/mdx.py @@ -1,16 +1,16 @@ from hashlib import md5 -from os.path import exists, join -from os import mkdir import re from markdown import Extension, etree, AtomicString from markdown.inlinepatterns import Pattern +from django.core.exceptions import ObjectDoesNotExist +from django.core.urlresolvers import reverse from django.template.defaultfilters import slugify from diggie.urls import RE_PAGE_NAME -from diggie.settings import MEDIA_DIR, MEDIA_URL -from diggie.math import math_to_png, MathException +from diggie.settings import IMAGES +from diggie.util import math_to_png, MathException # ============================================================================= # regular expressions @@ -34,10 +34,10 @@ class LinkPattern(Pattern): - def __init__(self, md): + def __init__(self, targets): Pattern.__init__(self, None) - self.md = md + self.targets = targets def getCompiledRegExp(self): @@ -49,7 +49,7 @@ name, anchor, alt = [s is not None and s.strip() for s in m.groups()[1:4]] anchor = anchor and "#%s" % slugify(anchor) or "" - self.md.targets.add(name) + self.targets.add(name) el = etree.Element('a') el.set("href", "/page/%s%s" % (name, anchor)) @@ -93,7 +93,7 @@ style = "" el = etree.Element('img') - el.set("src", "%simages/%s" % (MEDIA_URL, fname)) + el.set("src", IMAGES + fname) el.set("style", style) for key, value in [p.split("=", 1) for p in params if "=" in p]: el.set(key, value.strip("'\"")) @@ -108,7 +108,7 @@ # ----------------------------------------------------------------------------- -MATHERR = """The following math expression caused errors: +MATH_ERROR = """The following math expression caused errors: ------------------------------------------------------------------------------- %s ------------------------------------------------------------------------------- @@ -120,14 +120,12 @@ class MathPattern(Pattern): dpi = "120" - mathdir = join(MEDIA_DIR, "math") - mathurl = MEDIA_URL + "math/" - def __init__(self): + def __init__(self, math_all, math_rev): Pattern.__init__(self, None) - if not exists(self.mathdir): - mkdir(self.mathdir) + self.math_all = math_all + self.math_rev = math_rev def getCompiledRegExp(self): @@ -138,27 +136,26 @@ math, dpi = m.group(2), m.group(3) or self.dpi mhash = md5(math + dpi).hexdigest() - fpng = join(self.mathdir, "%s.png" % mhash) - fmeta = join(self.mathdir, "%s.meta" % mhash) - if exists(fpng): - with open(fmeta) as fp: - valign = int(fp.read().strip()) - else: + try: + fml = self.math_all.get(key=mhash) + except ObjectDoesNotExist: try: - valign = math_to_png(math, dpi, fpng) + img, valign = math_to_png(math, dpi) except MathException, e: el = etree.Element('pre') el.set('class', 'matherror') - el.text = AtomicString(MATHERR % (math, e, e.output)) + el.text = AtomicString(MATH_ERROR % (math, e, e.output)) return el - with open(fmeta, 'w') as fp: - fp.write(str(valign)) + img = img.encode("base64") + fml = self.math_all.create(key=mhash, img=img, valign=valign) + + self.math_rev.add(fml) el = etree.Element('img') - el.set('src', "%s%s.png" % (self.mathurl, mhash)) + el.set('src', reverse('diggie.views.math', args=(mhash,))) el.set('class', "math") - el.set('style', "vertical-align: %dpx" % valign) + el.set('style', "vertical-align: %dpx" % fml.valign) return el # ============================================================================= @@ -166,18 +163,36 @@ # ============================================================================= class DiggieExtension(Extension): + """A markdown extension for Diggie specific patterns.""" + + def __init__(self, math_all): + """Create a new extension instance. + + @param math_all: + A QuerySet for Math objects. Used to check for existing Math + objects and to create new ones. + + After markdown has been run, the attributes ``targets`` and + ``math_rev`` of this extension instance provide Diggie specific + processing results. ``targets`` is a set of page names linked to by the + processed revision. ``math_rev`` is a set of Math objects used in the + processed revision. + + """ + Extension.__init__(self, None) + + self.targets = set() # names of linked pages + self.math_all = math_all # all existing Math objects + self.math_rev = set() # Math objects in current revision def extendMarkdown(self, md, md_globals): - md.targets = set() - md.equations = set() - - lpatt = LinkPattern(md) + lpatt = LinkPattern(self.targets) md.inlinePatterns.add('diggie_link', lpatt, "%s)/?$' % RE_LABEL_NAME, 'diggie.views.pages'), (r'^page/(?P%s)/?$' % RE_PAGE_NAME, 'diggie.views.page'), (r'^page/(?P%s)/(?P\d+)/?$' % RE_PAGE_NAME, 'diggie.views.page'), + (r'^math/(?P[0-9a-fA-F]+)/?$', 'diggie.views.math'), (r'^edit/?$', 'diggie.views.edit'), (r'^edit/(?P%s)/?$' % RE_PAGE_NAME, 'diggie.views.edit'), (r'^edit/(?P%s)/(?P\d+)/?$' % RE_PAGE_NAME, 'diggie.views.edit'), diff --git a/app/diggie/util.py b/app/diggie/util.py --- a/app/diggie/util.py +++ b/app/diggie/util.py @@ -1,4 +1,87 @@ +from os.path import join import re +from shutil import rmtree +from subprocess import Popen, PIPE, STDOUT +import tempfile + +# ============================================================================= +# math rendering +# ============================================================================= + +_LATEX = r""" +\documentclass{article} +\usepackage{color} +\usepackage{amsmath} +\usepackage{amsfonts} +\usepackage[active,tightpage]{preview} +\definecolor{bg}{rgb}{ +1,1,1 +} +\definecolor{fg}{rgb}{ +0,0,0 +} +\pagestyle{empty} +\pagecolor{bg} +\begin{document} +\color{fg} +\begin{preview} +$ +%s +$ +\end{preview} +\end{document} +""" + +class MathException(Exception): + """Exception for math rendering errors.""" + + def __init__(self, msg, output): + super(MathException, self).__init__(msg) + self.output = output + +def math_to_png(source, dpi): + """Convert LaTeX math code to a PNG image. + + @param source: + latex math code + @param dpi: + image DPI + + @return: + the image as a data string and a CSS vertical alignment value for + baseline matching + + """ + def run(cmd): + try: + p = Popen(cmd, stdout=PIPE, stderr=STDOUT, cwd=tdir) + out = p.communicate()[0] + except OSError, e: + rmtree(tdir) + raise MathException("failed to run *%s*" % cmd[0], e) + if p.returncode != 0: + rmtree(tdir) + raise MathException("*%s* had problems" % cmd[0], out) + return out + + tdir = tempfile.mkdtemp() + with open(join(tdir, "math.latex"), 'w') as fp: + fp.write(_LATEX % source) + cmd = ["latex", "-interaction=nonstopmode", "math.latex"] + run(cmd) + cmd = ["dvipng", "-D", dpi, "-z", "9", "--depth*", "-o", "math.png", + "math.dvi"] + output = run(cmd) + depth = (re.findall(r'depth=([0-9]+)', output) + [0])[0] + valign = -int(depth) + with open(join(tdir, "math.png")) as fp: + img = fp.read() + rmtree(tdir) + return img, valign + +# ============================================================================= +# parse pragmas at the beginning of a document +# ============================================================================= RX_PRAGMA = re.compile(r'^#([a-z]+)(?: +(.*?) *)?$', re.IGNORECASE) diff --git a/app/diggie/views.py b/app/diggie/views.py --- a/app/diggie/views.py +++ b/app/diggie/views.py @@ -2,19 +2,21 @@ import re from django.db.models import Q -from django.http import HttpResponseRedirect +from django.http import HttpResponseRedirect, HttpResponse, HttpResponseNotFound from django.shortcuts import render_to_response from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse from django.views.decorators.cache import cache_page, never_cache from django.conf import settings -from diggie.models import Page, Label, Revision +from diggie.models import Page, Label, Revision, Math from diggie.util import xpragmas from diggie.urls import RE_LABEL_NAME from diggie.settings import WIKINAME A_VERY_LONG_TIME = 31536000 # one year +A_WHILE = 300 # 5 minutes +A_MOMENT = 60 # 1 minute HIT_COUNT_DELAY = timedelta(seconds=60) class count_hits(object): @@ -73,8 +75,8 @@ val = choices[0] return val -@never_cache # no upstream caching -@cache_page(60) # server-side caching +@never_cache # no upstream caching +@cache_page(A_MOMENT) # server-side caching def main(request): _dview("main", request) @@ -115,8 +117,8 @@ return render_to_response('diggie/labels.html', rqc) -@never_cache # no upstream caching -@cache_page(60) # server-side caching +@never_cache # no upstream caching +@cache_page(A_MOMENT) # server-side caching def pages(request, label=None): _dview("pages", request) @@ -227,8 +229,19 @@ return render_to_response('diggie/page.html', rqc) -@never_cache # no upstream caching -@cache_page(60) # server-side caching +@cache_page(A_WHILE) # server-side caching + upstream caching +def math(request, key): + _dview("math", request) + + try: + fml = Math.objects.get(key=key) + except ObjectDoesNotExist: + return HttpResponseNotFound() + + return HttpResponse(fml.img.decode("base64"), mimetype="image/png") + +@never_cache # no upstream caching +@cache_page(A_MOMENT) # server-side caching def history(request, name=None): _dview("history", request) @@ -267,16 +280,16 @@ %s """ -PAGE_EXAMPLE = """Section 1 +PAGE_EXAMPLE = """Section ========= -Section 1.2 ------------ +Subsection +---------- Here comes a *nice* list: - * item 1 - * item 2 + - item `1` + - item **2** """ def edit(request, name="", revn="-1"):