# Copyright (c) 2001 Samuel Brauer. All Rights Reserved. NO WARRANTY.
# $Revision: 1.24 $
import codecs
import cPickle
import cStringIO
import os
import fnmatch
import re
import string
import sys
import time
import traceback
import types
import urllib
import xml.sax
import xml.sax.handler
import xml.sax.xmlreader
import spb.makiLogic
import spb.XmlUtil
import spb.makiVars
import spb.makiExceptions
THREADS = 1
try: import thread
except: THREADS = 0
# FIXME: if THREADS, be sure to behave in a threadsafe manner.
# not everyone may have/use 4xslt
try:
import Ft.Xml.Xslt.Processor, Ft.Xml.InputSource
Ft_processor = Ft.Xml.Xslt.Processor.Processor()
except:
pass
# not everyone may have/use Sablot
try: import Sablot
except: pass
# not everyone may have/use libxsltmod
try: import libxsltmod
except: pass
# not everyone may have/use libxslt
LIBXSLT = 1
try: import libxml2, libxslt
except: LIBXSLT = 0
# not everyone may have/use Pyana
try: import Pyana
except: pass
###########################################################
# Global constants
###########################################################
MAKI_DIR = spb.makiVars.MAKI_DIR
SW_VERSION = spb.makiVars.VERSION
CONFIG_FILE = MAKI_DIR + os.sep + "maki-conf.xml"
CACHE_DIR = MAKI_DIR + os.sep + "cache"
ERROR_FILE = MAKI_DIR + os.sep + "maki.log"
SW_NAME = "maki"
SW_URL = "http://maki.sourceforge.net/"
MAKI_NS = "http://maki.sourceforge.net/maki"
DEBUGLEVEL = 2
KNOWN_TRANSFORMERS = ('4xslt', 'Sablot', 'libxsltmod', 'libxslt', 'Pyana')
DEFAULT_PROCESS_MODULE_NAME = "spb.makiLogic"
DEFAULT_PROCESS_FUNCTION_NAME = "process"
KNOWN_CONFIG_ELEMENTS = (
'/config',
'/config/fancy-error-page',
'/config/debug-level',
'/config/default-transformer',
'/config/alias',
'/config/zone',
'/config/rule',
'/config/rule/stylesheet',
'/config/rule/process',
'/config/rule/write',
'/config/rule/debug')
KNOWN_ZONE_ELEMENTS = (
'/zone',
'/zone/rule',
'/zone/rule/stylesheet',
'/zone/rule/process',
'/zone/rule/write',
'/zone/rule/debug')
error_file = None
def log(s):
error_file.write("[%s] %s\n" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), s))
error_file.flush()
def flushLog():
error_file.flush()
def error(e, stacktrace=1):
log("error: " + str(e))
if(isinstance(e, xml.sax._exceptions.SAXParseException)):
log("at line %d column %d" % (e._linenum, e._colnum))
if(stacktrace):
traceback.print_exc(None, error_file)
flushLog()
def warn(msg, e=None):
if(msg): log("warn: " + msg)
if(e):
log("warn: " + str(e))
traceback.print_exc(None, error_file)
flushLog()
def debug(msg):
if(msg): log("debug: " + msg)
try:
error_file = open(ERROR_FILE, 'a', 1)
except Exception, e:
error_file = sys.stderr
warn("Couldn't open error log (%s)!" % ERROR_FILE, e)
# display banner upon module load
#log("""%s\n%s\nCopyright (c) 2001 Samuel Brauer. All Rights Reserved. NO WARRANTY.\n%s""" % ('-'*70, SW_NAME, '-'*70))
def debugWithLineNumbers(text):
debug("--- DEBUG START ---")
lineno = 0
for line in text.split("\n"):
lineno += 1
debug("%4d: %s" % (lineno, line))
debug("--- DEBUG END ---")
def htmlize(s):
return string.replace(string.replace(s, "&", "&"), "<", "<")
def MAKI_return():
raise spb.makiExceptions.MakiReturn
def MAKI_useCache():
raise spb.makiExceptions.MakiUseCache
def htmlErrorPage(trans, e=None):
buff = cStringIO.StringIO()
buff.write("""<html><head><title>Error</title></head>
<body bgcolor='white'><center><br><br>
<table border='0' cellspacing='0' cellpadding='0' bgcolor='black'>
<tr><td><table width='100%' border='0' cellspacing='1' cellpadding='5'>\n""")
buff.write("<tr bgcolor='#ff9900'><td align='center'><b><font size='+1' color='white'>"+("<a href='%s'>%s</a> (ver %s) encountered an error" % (SW_URL, SW_NAME, SW_VERSION))+"</font></b></td></tr>\n")
if(e):
buff.write("<tr bgcolor='white'><td>"+str(e.__class__)+':')
if(hasattr(e, 'args')):
buff.write("<ul>\n")
if(isinstance(e.args, types.StringType)):
buff.write("<li>"+htmlize(str(e.args))+"</li>\n")
elif(isinstance(e.args, types.UnicodeType)):
buff.write("<li>"+htmlize(str(e.args.encode("utf-8")))+"</li>\n")
else:
for arg in e.args:
buff.write("<li>"+htmlize(str(arg))+"</li>\n")
if(isinstance(e, xml.sax._exceptions.SAXParseException)):
buff.write("<li>at line %d column %d</li>\n" % (e._linenum, e._colnum))
buff.write("</ul>\n")
else:
buff.write('<br>'+htmlize(str(e)))
buff.write("<br><i>Debugging traceback:</i><br><code>\n")
traceback_buff = cStringIO.StringIO()
traceback.print_tb(sys.exc_traceback, file=traceback_buff)
buff.write(string.replace(htmlize(traceback_buff.getvalue()), "\n", "<br>\n"))
traceback_buff.close()
buff.write("</code></td></tr>\n")
buff.write("</table></td></tr></table></center></body></html>\n")
trans.writeResponseOutput(buff.getvalue())
buff.close()
class Step:
def __init__(self, stepType, attributes):
"""
stepType is a string naming the type of this step ('stylesheet', 'process', 'debug', 'write')
attributes is a dict of name/value pairs
"""
if(stepType == 'stylesheet'):
self.file = None
self.transformer = defaultTransformer
self.use_pi = 0
self.debug = 0
for attname in attributes.keys():
if(attname == 'file'):
self.file = attributes[attname]
elif(attname == 'transformer'):
self.transformer = attributes[attname]
elif(attname == 'use-pi'):
if(attributes[attname] == 'yes'):
self.use_pi = 1
elif(attname == 'debug'):
if(attributes[attname] == 'yes'):
self.debug = 1
else:
raise RuntimeError, "Unrecognized attribute for %s: %s" % (stepType, attname)
elif(stepType == 'process'):
cacheUnit = 'seconds'
self.module = DEFAULT_PROCESS_MODULE_NAME
self.function = DEFAULT_PROCESS_FUNCTION_NAME
self.cacheOnParms = 1
self.cacheOnCookies = []
self.cacheOnEnviron = []
# cacheTime may be "never", "forever", "dynamic", or a number of seconds (or some other unit of time)
self.cacheTime = "never" # by default, do not do any caching
for attname in attributes.keys():
if(attname == 'module'):
self.module = attributes[attname]
elif(attname == 'function'):
self.function = attributes[attname]
elif(attname == 'cacheOnParms'):
attval = attributes[attname]
if(attval == 'no'):
self.cacheOnParms = 0
elif(attname == 'cacheOnCookies'):
attval = attributes[attname]
if(attval): self.cacheOnCookies = string.split(attval, ';')
elif(attname == 'cacheOnEnviron'):
attval = attributes[attname]
if(attval): self.cacheOnEnviron = string.split(attval, ';')
elif(attname == 'cacheTime'):
attval = attributes[attname]
if(attval in ('never', 'forever', 'dynamic')):
self.cacheTime = attval
else:
self.cacheTime = int(attval)
elif(attname == 'cacheUnit'):
cacheUnit = attributes[attname]
if(cacheUnit not in ('seconds', 'minutes', 'hours', 'days')):
raise RuntimeError, "Unrecognized value for @cacheUnit: %s" % cacheUnit
else:
raise RuntimeError, "Unrecognized attribute for %s: %s" % (stepType, attname)
if(cacheUnit == 'minutes'):
self.cacheTime *= 60
elif(cacheUnit == 'hours'):
self.cacheTime *= 3600
elif(cacheUnit == 'days'):
self.cacheTime *= 86400
elif(stepType == 'write'):
self.contentType = "text/html"
for attname in attributes.keys():
if(attname == 'contentType'):
self.contentType = str(attributes[attname])
else:
raise RuntimeError, "Unrecognized attribute for %s: %s" % (stepType, attname)
elif(stepType == 'debug'):
self.parse = 0
for attname in attributes.keys():
if(attname == 'parse'):
if(attributes[attname] == 'yes'):
self.parse = 1
else:
raise RuntimeError, "Unrecognized attribute for %s: %s" % (stepType, attname)
else:
raise RuntimeError, "Unrecognized step type: "+stepType
self.stepType = stepType
def getCode(self):
"""return a string representing this step to be used as part of cache key"""
if(self.stepType == 'stylesheet'):
sheetname = "None"
if(self.file):
sheetname = string.replace(self.file, os.sep, '-')
return 'stylesheet-%s' % sheetname
elif(self.stepType == 'process'):
return 'process-%s-%s' % (self.module, self.function)
else:
return ''
class Rule:
def __init__(self, matchType, pattern):
"""
'matchType' is either 'fn' (filename match) or 're' (regular expression).
'pattern' is a string representing either a fnmatch pattern or a regular expression (depending on value of matchType).
to be used to match the real path value of a request.
"""
self.matchType = matchType
self.pattern = pattern
if(self.matchType == 're'):
if(os.name == 'nt'):
# ignore case of filenames on Windows
self.re = re.compile(self.pattern, re.I)
else:
self.re = re.compile(self.pattern)
self.steps = []
def addStep(self, step):
self.steps.append(step)
def match(self, path):
if(self.matchType == 're'): return self.re.match(path)
else: return fnmatch.fnmatch(path, self.pattern)
class Zone:
def __init__(self, path, configfile):
"""
'path' is a string representing the absolute path that this zone
contains rules for.
'configfile' is the full path to the config file for this zone.
"""
if(os.name == 'nt'):
# ignore case of filenames on Windows
self.path = path.lower()
else:
self.path = path
self.configfile = configfile
self.configLastRead = 0
self.rulelist = []
def clearRules(self):
self.rulelist = []
def addRule(self, rule):
self.rulelist.insert(0, rule)
def getRulelist(self):
return self.rulelist
def match(self, path):
if(os.name == 'nt'):
# ignore case of filenames on Windows
return path.lower().startswith(self.path)
else:
return path.startswith(self.path)
class MakiSablotMsgHandler:
def makeCode(self, severity, facility, code):
pass
def log(self, code, level, fields):
pass
def error(self, code, level, fields):
msg = line = uri = ''
for field in fields:
if(field[0:4] == 'msg:'): msg = field[4:]
elif(field[0:5] == 'line:'): line = field[5:]
elif(field[0:4] == 'URI:'): uri = field[4:]
context = 'XML input'
if(uri == 'arg:/xsl'): context = 'stylesheet'
raise RuntimeError, "Sablot error at line %s of %s: %s" % (line, context, msg)
if(LIBXSLT):
class libxsltError(Exception):
def __init__(self, args=None):
self.args = args
class MakiLibxsltErrorHandler:
def __init__(self):
self.err_msgs = []
def errorCallback(self, ctx, str):
err_msg = "%s %s" % (ctx, str)
self.err_msgs.append(err_msg)
def getErrorMsgs(self):
return self.err_msgs
def getErrorMsgString(self):
return string.join(self.err_msgs, "")
def reset(self):
self.err_msgs = []
# Since the registerErrorHandler functions are global, we
# can only have one error_handler.
# FIXME: If THREADS, be sure to
# treat it in a threadsafe manner.
libxslt_error_handler = MakiLibxsltErrorHandler()
libxml2.registerErrorHandler(libxslt_error_handler.errorCallback, "")
libxslt.registerErrorHandler(libxslt_error_handler.errorCallback, "")
libxml2.lineNumbersDefault(1)
libxml2.substituteEntitiesDefault(1)
def normalizeFilepath(fn):
if(not os.path.isabs(fn)):
fn = os.path.abspath(fn)
#return os.path.normpath(fn)
return os.path.normpath(fn).replace("\\", "/") # for Windows
###########################################################
# Global variables
###########################################################
configPath = normalizeFilepath(CONFIG_FILE)
zones = []
configLastRead = 0
persistDict = {}
defaultTransformer = None
aliases = {}
fancyErrorPage = 1
###########################################################
def superquote(s):
return string.replace(urllib.quote(s), '/', '%2f')
def createDirectoryLevels(fn):
dir = os.path.dirname(fn)
if(not os.path.exists(dir)):
os.makedirs(dir)
def getCachedFilename(filename):
#return CACHE_DIR + filename
return CACHE_DIR + filename.replace(':', '_') # for Windows
class StylesheetPIHandler(spb.XmlUtil.PrintHandler):
"""SAX handler used to find (and strip) the first xml-stylesheet PI
from a document. Used by stylesheet steps where @use-pi is 'yes'."""
def __init__(self, out):
spb.XmlUtil.PrintHandler.__init__(self, out)
self.href = None
def getHref(self):
return self.href
def processingInstruction(self, target, data):
if(target == 'xml-stylesheet' and not self.href):
items = string.split(data)
# Graham Higgins <gjh@bel-epa.com> pointed out that
# we should only use those PIs where type='text/xml'
# Then Martin Dalum (garfield@sunsite.dk) pointed
# out that text/xsl should also be allowed.
type = None
for item in items:
if(item.startswith("type=")):
# strip 'type=' and the leading/trailing quotes
type = item[6:-1]
if(type in ("text/xml", "text/xsl")):
for item in items:
if(item.startswith("href=")):
# strip 'href=' and the leading/trailing quotes
self.href = item[6:-1]
return
spb.XmlUtil.PrintHandler.processingInstruction(self, target, data)
class IncludeHandler(xml.sax.saxlib.ContentHandler):
def __init__(self, out, hreflist, fileroot):
self.out = out
self.hreflist = hreflist
self.stack = []
self.stack.append([fileroot, 0, 0]) # (fileroot, skipElementCount, openElementCount)
self.printhandler = spb.XmlUtil.PrintHandler(self.out)
def startPrefixMapping(self, prefix, uri):
self.printhandler.startPrefixMapping(prefix, uri)
def endPrefixMapping(self, prefix):
self.printhandler.endPrefixMapping(prefix)
def startDocument(self):
if(len(self.stack) == 1): self.printhandler.startDocument()
def endDocument(self):
self.stack.pop()
def startElementNS(self, name, qname, attrs):
self.stack[-1][2] += 1 # increment openElementCount
if(self.stack[-1][2] > self.stack[-1][1]): # if openElementCount > skipElementCount
if(name[0] == MAKI_NS and name[1] == 'include'):
href = attrs.get((None, 'href'), None)
if(not href): raise RuntimeError, "<maki:include> without @href"
keepRoot = attrs.get((None, 'keepRoot'), None)
if(keepRoot == 'no'): skip = 1
else: skip = 0
if(href[0] != '/'): href = self.stack[-1][0] + os.sep + href
href = normalizeFilepath(href)
if(href in self.hreflist):
#raise RuntimeError, "Circular include of %s detected." % href
warn("Circular include of %s detected." % href)
else:
if(DEBUGLEVEL > 1): debug("href=%s" % href)
self.hreflist.append(href)
self.stack.append([os.path.dirname(href), skip, 0])
data = spb.XmlUtil.getUTF8StringFromFilename(href)
spb.XmlUtil.parseString(data, self, 1)
else:
self.printhandler.startElementNS(name, qname, attrs)
def endElementNS(self, name, qname):
if(self.stack[-1][2] > self.stack[-1][1]): # if openElementCount > skipElementCount
if(name[0] == MAKI_NS and name[1] == 'include'):
pass
else:
self.printhandler.endElementNS(name, qname)
if(self.stack): self.stack[-1][2] -= 1 # decrement openElementCount
def characters(self, content):
self.printhandler.characters(content)
def ignorableWhitespace(self, content):
self.printhandler.ignorableWhitespace(content)
def processingInstruction(self, target, data):
self.printhandler.processingInstruction(target, data)
def handleIncludes(data, filename):
"""Search thru data for any elements named "maki:include" and note
the values of their @href. Normalize the filenames of each @href and
store list in a dependency cache file.
Additionally, expand any <maki:include href="X" keepRoot='yes|no'/>
occurrences that we encounter. """
if(DEBUGLEVEL > 1): debug("running handleIncludes() for %s" % filename)
fileroot = os.path.dirname(filename)
buffer = cStringIO.StringIO()
hreflist = []
handler = IncludeHandler(buffer, hreflist, fileroot)
spb.XmlUtil.parseString(data, handler, 1)
if(DEBUGLEVEL > 1): debug("hreflist="+repr(hreflist))
writeCachedData(os.sep + "dependencies" + os.sep + filename, hreflist, 1)
data = buffer.getvalue()
buffer.close()
return data
def removeCachedData(filename):
if(DEBUGLEVEL > 1): debug("entered removeCachedData('%s')" % filename)
cachedFilename = getCachedFilename(filename)
if(os.path.exists(cachedFilename)):
os.remove(cachedFilename)
else:
if(DEBUGLEVEL > 1): debug("cached data file does not exist")
def getCachedMtime(filename):
cachedFilename = getCachedFilename(filename)
if(DEBUGLEVEL > 1): debug("cachedFilename=%s" % cachedFilename)
if(not os.path.exists(cachedFilename)):
if(DEBUGLEVEL > 1): debug("cached data file does not exist")
return 0
if(DEBUGLEVEL > 1): debug("cached data file exists; get mtime...")
return os.path.getmtime(cachedFilename)
# FIXME: add file locking to readCachedData() and writeCachedData()
def readCachedData(filename, pickleFlag=0):
cachedFilename = getCachedFilename(filename)
if(not os.path.exists(cachedFilename)):
if(DEBUGLEVEL > 1): debug("cached data file does not exist")
return None
if(DEBUGLEVEL > 1): debug("cached data file exists; try to use it...")
try:
if(pickleFlag):
cachefile = open(cachedFilename)
try:
data = cPickle.load(cachefile)
finally:
cachefile.close()
else:
data = spb.XmlUtil.getUTF8StringFromFilename(cachedFilename)
if(DEBUGLEVEL > 1): debug("successfully read cached file")
except Exception, e:
warn("unable to read from cache: "+cachedFilename, e)
data = None
return data
def writeCachedData(filename, data, pickleFlag=0):
cachedFilename = getCachedFilename(filename)
if(DEBUGLEVEL > 1): debug("about to try to write cached data to file...")
cachefile = None
try:
createDirectoryLevels(cachedFilename)
if(data == None):
cachefile = open(cachedFilename, 'w')
warn("writeCachedData() called with data=None")
pass # nothing to write
elif(pickleFlag):
cachefile = open(cachedFilename, 'w')
cPickle.dump(data, cachefile, 1)
else:
if(isinstance(data, types.StringType)):
cachefile = open(cachedFilename, 'w')
else:
cachefile = codecs.open(cachedFilename, 'w', spb.XmlUtil.getEncodingFromString(data))
cachefile.write(data)
cachefile.close()
cachefile = None
if(DEBUGLEVEL > 1): debug("successfully wrote cache file")
except Exception, e:
warn("unable to write to cache: "+cachedFilename, e)
try:
if(cachefile): cachefile.close()
except:
pass
def getFileMtime(filename, fileroot=None):
"""returns greatest mtime of file or files it includes"""
if(fileroot): filename = fileroot + os.sep + filename
filename = normalizeFilepath(filename)
if(DEBUGLEVEL > 1): debug("entered getFileMtime('%s')" % filename)
if(not os.path.exists(filename)):
if(DEBUGLEVEL > 1): debug("file does not exist")
return 0
if(DEBUGLEVEL > 1): debug("file exists; get mtime...")
mtime = os.path.getmtime(filename)
# now see if any dependencies have a more recent mtime...
if(DEBUGLEVEL > 1): debug("getting deplist...")
deplist = readCachedData(os.sep + "dependencies" + os.sep + filename, 1)
if(deplist == None):
return mtime
for dep in deplist:
if(not os.path.exists(dep)): continue
dep_mtime = getFileMtime(dep)
if(dep_mtime > mtime):
mtime = dep_mtime
return mtime
def getFile(filename, fileroot=None):
"""This function reads the contents from a specified filename, possibly expanding includes. """
if(DEBUGLEVEL > 1): debug("entered getFile(); filename=%s and fileroot=%s" % (filename, str(fileroot)))
if(fileroot): filename = fileroot + os.sep + filename
filename = normalizeFilepath(filename)
if(not os.path.exists(filename)):
raise RuntimeError, "File not found: %s" % filename
if(DEBUGLEVEL > 1): debug("about to try to open file...")
try:
data = spb.XmlUtil.getUTF8StringFromFilename(filename)
except Exception, e:
msg = "Caught an exception while reading %s" % filename
warn(msg, e)
raise RuntimeError, "While reading %s, caught an exception: %s: %s" % (filename, str(e.__class__), str(e))
# search thru document for dependencies on other files
if(data.find("maki:include") > -1):
data = handleIncludes(data, filename)
if(DEBUGLEVEL > 1): debug("successfully read file")
# write to cache file
writeCachedData(os.sep + "input" + os.sep + filename, data, 0)
return data
def getRule(path_translated):
"""Given a full path to a file, return the rule that should be used
to process the file. While doing this, check if the relevent zone config
file has changed and needs to be re-parsed. """
global zones
if(DEBUGLEVEL > 1): debug("searching for rule to match: "+path_translated)
for zone in zones:
if(DEBUGLEVEL > 1): debug("checking zone with path='"+zone.path+"'")
if(zone.match(path_translated)):
if(DEBUGLEVEL > 1): debug("zone matches!")
# re-read zone config file if necessary
if(zone.configfile):
if(not os.path.exists(zone.configfile)):
raise RuntimeError, "Zone config file does not exist: "+zone.configfile
file_mtime = getFileMtime(zone.configfile)
if(zone.configLastRead >= file_mtime):
if(DEBUGLEVEL > 1): debug("using cached zone/rule config")
else:
if(DEBUGLEVEL > 1): debug("about to try to load config data...")
data = getFile(zone.configfile)
handler = ZoneHandler(zone)
spb.XmlUtil.parseString(data, handler, 1)
if(DEBUGLEVEL > 1): debug("successfully read zone config data")
zone.configLastRead = file_mtime
# strip zone path from front of path_translated
relpath = path_translated[len(zone.path):]
for rule in zone.getRulelist():
if(DEBUGLEVEL > 1): debug("checking rule with pattern='"+rule.pattern+"'")
if(rule.match(relpath)):
if(DEBUGLEVEL > 1): debug("rule matches!")
return rule
# if the zone we just looked it isn't the default zone,
# try looking in the default zone
if(zone.path):
zone = zones[-1]
if(DEBUGLEVEL > 1): debug("checking in default zone")
for rule in zone.getRulelist():
if(DEBUGLEVEL > 1): debug("checking rule with pattern='"+rule.pattern+"'")
if(rule.match(path_translated)):
if(DEBUGLEVEL > 1): debug("rule matches!")
return rule
if(DEBUGLEVEL > 1): debug("no match found :(")
return None
def process(trans, timein):
global defaultTransformer
xml_filename = normalizeFilepath(trans.getRealPath())
if(DEBUGLEVEL > 0): debug("XML_FILENAME: "+str(xml_filename))
query_string = trans.getEnvironValue("QUERY_STRING") # may be None
canUseCache = 1
cacheable = 1
fileroot = os.path.dirname(xml_filename)
xml_data = None
rule = getRule(xml_filename)
if(not rule): raise RuntimeError, "No rule matches %s" % xml_filename
parmsMatter = 0
parmString = ''
if(query_string): parmString = query_string
cacheKey = xml_filename
lastGoodFilename = os.sep + "output" + os.sep + cacheKey + os.sep + '__DOC__'
sourceMtime = getFileMtime(xml_filename)
cachedMtime = getCachedMtime(lastGoodFilename)
if(DEBUGLEVEL > 0): debug("CHECKING XML FILE: cachedMtime=%d sourceMtime=%d" % (cachedMtime, sourceMtime))
if(cachedMtime <= sourceMtime):
canUseCache = 0
xml_data = getFile(xml_filename)
writeCachedData(lastGoodFilename, xml_data, 0)
evalDict = {}
# Add some global variables for use by process steps
evalDict["MAKI_trans"] = trans # the request
evalDict["MAKI_request"] = trans # DEPRECATED
evalDict["MAKI_persistDict"] = persistDict # a dictionary that persists across requests, for caching
evalDict["MAKI_log"] = log # our debug function, which pages can use to write debugging messages to the maki error log
evalDict["MAKI_return"] = MAKI_return # a process step can call this function to exit out of a rule early
evalDict["MAKI_useCache"] = MAKI_useCache # a process step can call this function to exit out of a process step early and signify that cached output can be used
steps = rule.steps
for stepIdx in range(len(steps)):
step = steps[stepIdx]
stepcode = step.getCode()
if(stepcode): cacheKey += os.sep + stepcode
if(step.stepType == 'process' and not(parmsMatter) and step.cacheOnParms):
parmsMatter = 1
if(parmString):
cacheKey += os.sep
cacheKey += parmString
if(step.stepType == 'process'):
if(step.cacheOnCookies):
cacheKey += os.sep
for name in step.cacheOnCookies:
if(cacheKey[-1] != os.sep): cacheKey += '&'
cacheKey += '%s=%s' % (name, superquote(str(trans.getCookieValue(name))))
if(step.cacheOnEnviron):
cacheKey += os.sep
for name in step.cacheOnEnviron:
if(cacheKey[-1] != os.sep): cacheKey += '&'
cacheKey += '%s=%s' % (name, superquote(str(trans.getEnvironValue(name))))
cacheKeyFile = cacheKey + os.sep + '__DOC__'
if(step.stepType == 'debug'):
needToNullOut=0
if(not xml_data):
needToNullOut=1
xml_data = readCachedData(lastGoodFilename, 0)
debug_out = xml_data
if(isinstance(xml_data, types.UnicodeType)):
debug_out = xml_data.encode('utf-8')
debugWithLineNumbers(debug_out)
if(step.parse):
try:
spb.XmlUtil.parseString(xml_data)
debug("Parse OK.")
except Exception, e:
debug("Parse failed.")
error(e, 0)
if(needToNullOut): xml_data = None
elif(step.stepType == 'write'):
if(not xml_data): xml_data = readCachedData(lastGoodFilename, 0)
write_out = xml_data
if(isinstance(xml_data, types.UnicodeType)):
write_out = xml_data.encode('utf-8')
trans.writeResponseOutput(write_out, step.contentType)
elif(step.stepType == 'stylesheet'):
steptimein = time.time()
if(DEBUGLEVEL > 0): debug("CACHE FILE: "+cacheKeyFile)
if(step.use_pi):
needToNull = 0
if(not xml_data):
xml_data = readCachedData(lastGoodFilename, 0)
needToNull = 1
stylesheet = None
# Search thru document for xml-stylesheet PI.
# If found, use the value of its href "attribute".
# Also, remove the PI from the document.
buffer = cStringIO.StringIO()
handler = StylesheetPIHandler(buffer)
spb.XmlUtil.parseString(xml_data, handler, 1)
stylesheet = handler.getHref()
if(not stylesheet): raise RuntimeError, "Could not find xml-stylesheet href in document."
if(needToNull): xml_data = None # strange, but this seems to be necessary
else: xml_data = buffer.getvalue()
buffer.close()
else:
stylesheet = step.file
if(not stylesheet):
basename = os.path.basename(xml_filename)
if(basename.find('.') > 0):
stylesheet = os.path.splitext(basename)[0] + ".xsl"
else:
stylesheet = basename + ".xsl"
#if(stylesheet[0] != os.sep):
if(not os.path.isabs(stylesheet)): # for Windows
stylesheet = fileroot + os.sep + stylesheet
stylesheet = normalizeFilepath(stylesheet)
if(DEBUGLEVEL > 0): debug("stylesheet="+stylesheet)
sourceMtime = getFileMtime(stylesheet)
cachedMtime = getCachedMtime(os.sep + "output" + os.sep + cacheKeyFile)
if(DEBUGLEVEL > 0): debug("CHECKING STYLESHEET FILE: cachedMtime=%d sourceMtime=%d" % (cachedMtime, sourceMtime))
if(canUseCache and cachedMtime > sourceMtime):
lastGoodFilename = os.sep + "output" + os.sep + cacheKeyFile
else:
canUseCache = 0
if(not xml_data): xml_data = readCachedData(lastGoodFilename, 0)
transformer = step.transformer
if(DEBUGLEVEL > 0): debug("APPLYING STYLESHEET (using %s)..." % transformer)
baseURI = 'file://'+os.path.dirname(stylesheet)+'/'
if(DEBUGLEVEL > 1): debug("BASE URI: "+baseURI)
cachedInputMtime = getCachedMtime(os.sep + "input" + os.sep + stylesheet)
if(cachedInputMtime > sourceMtime):
if(DEBUGLEVEL > 0): debug("using cached stylesheet")
xsl_data = readCachedData(os.sep + "input" + os.sep + stylesheet, 0)
else:
if(DEBUGLEVEL > 0): debug("reading stylesheet")
xsl_data = getFile(stylesheet)
if(isinstance(xml_data, types.UnicodeType)):
xml_data = xml_data.encode('utf-8')
if(isinstance(xsl_data, types.UnicodeType)):
xsl_data = xsl_data.encode('utf-8')
if(step.debug): debugWithLineNumbers(xsl_data)
try:
if(transformer == 'libxsltmod'):
# FIXME: how to set base URI with libxsltmod?
# FIXME: write a messageHandler class for libxsltmod1.3a
xml_data = libxsltmod.translate_to_string('s', xsl_data, 's', xml_data, None)
elif(transformer == 'libxslt'):
# FIXME: how to set base URI with libxslt?
styledoc = None
style = None
datadoc = None
result = None
try:
# FIXME: if THREADS, lock on the libxslt_error_handler
libxslt_error_handler.reset()
styledoc = libxml2.parseDoc(xsl_data)
if(libxslt_error_handler.getErrorMsgs()): raise libxsltError, libxslt_error_handler.getErrorMsgs()
style = libxslt.parseStylesheetDoc(styledoc)
if(libxslt_error_handler.getErrorMsgs()): raise libxsltError, libxslt_error_handler.getErrorMsgs()
datadoc = libxml2.parseDoc(xml_data)
if(libxslt_error_handler.getErrorMsgs()): raise libxsltError, libxslt_error_handler.getErrorMsgs()
result = style.applyStylesheet(datadoc, None)
if(libxslt_error_handler.getErrorMsgs()): raise libxsltError, libxslt_error_handler.getErrorMsgs()
xml_data = result.serialize()
finally:
# FIXME: if THREADS, release lock on the libxslt_error_handler
if(result): result.freeDoc()
if(datadoc): datadoc.freeDoc()
if(style): style.freeStylesheet()
elif(styledoc): styledoc.freeDoc()
elif(transformer == 'Sablot'):
processor = Sablot.CreateProcessor()
processor.setBase(baseURI)
processor.regHandler(Sablot.HLR_MESSAGE, MakiSablotMsgHandler())
processor.run('arg:xsl', 'arg:xml', 'arg:output', [], [('xml', xml_data), ('xsl', xsl_data)])
xml_data = processor.getResultArg('output')
processor.freeResultArgs()
del processor
elif(transformer == 'Pyana'):
# FIXME: how to set base URI with Pyana?
xml_data = Pyana.transform2String(source=xml_data, style=xsl_data)
elif(transformer == '4xslt'):
Ft_processor.appendStylesheet(Ft.Xml.InputSource.DefaultFactory.fromString(xsl_data, baseURI))
xml_data = Ft_processor.run(Ft.Xml.InputSource.DefaultFactory.fromString(xml_data, "uri"), ignorePis=1)
Ft_processor.reset()
else:
raise RuntimeError, "Unrecognized transformer: "+str(transformer)
except:
# the old cache file is no good, so remove it
oldCacheFilename = os.sep + "output" + os.sep + cacheKeyFile
removeCachedData(oldCacheFilename)
raise
if(cacheable):
# try to write xml_data to cachefile
writeCachedData(os.sep + "output" + os.sep + cacheKeyFile, xml_data, 0)
if(DEBUGLEVEL > 0): debug("TIME FOR STYLESHEET STEP: "+str(time.time() - steptimein) + " seconds")
elif(step.stepType == 'process'):
steptimein = time.time()
if(DEBUGLEVEL > 0): debug("CACHE FILE: "+cacheKeyFile)
cachedMtime = getCachedMtime(os.sep + "output" + os.sep + cacheKeyFile)
if(DEBUGLEVEL > 0): debug("CHECKING PROCESS FILE: cachedMtime=%d" % cachedMtime)
if(step.cacheTime == "never"):
canUseCache = 0
cacheable = 0
if(DEBUGLEVEL > 1): debug("cacheable=%d" % cacheable)
if(DEBUGLEVEL > 1): debug("canUseCache=%d" % canUseCache)
if(DEBUGLEVEL > 0):
debug("step.cacheTime=%s currtime - cachedMtime = %d" % (step.cacheTime, time.time() - cachedMtime))
if(canUseCache and ((step.cacheTime == "forever" and cachedMtime > 0) or ((isinstance(step.cacheTime, types.IntType) and step.cacheTime > 0 and (time.time() - cachedMtime < step.cacheTime))))):
lastGoodFilename = os.sep + "output" + os.sep + cacheKeyFile
else:
evalDict["MAKI_cacheTime"] = 0
if(canUseCache and step.cacheTime == "dynamic"): evalDict["MAKI_cacheTime"] = cachedMtime
canUseCache = 0
if(DEBUGLEVEL > 0): debug("RUNNING PROCESS...")
if(not xml_data): xml_data = readCachedData(lastGoodFilename, 0)
exec("import "+step.module, globals())
myfunc = eval(step.module+'.'+step.function, globals())
outbuffer = cStringIO.StringIO()
# If your process has some sort of cleanup that it absolutely
# must do, it can register a cleanup function and
# we will make sure that it is run.
evalDict["MAKI_cleanup"] = None
try:
try:
myfunc(xml_data, outbuffer, evalDict)
finally:
cleanup= evalDict.get("MAKI_cleanup", None)
if(not cleanup): cleanup = evalDict.get("MAKI_cleanupFunction", None) # MAKI_cleanupFunction is deprecated
if(cleanup):
evalDict["MAKI_cleanup"] = None
try:
cleanup()
except Exception, e:
log("Caught an exception while applying cleanup:")
error(e)
xml_data = outbuffer.getvalue()
if(cacheable):
# try to write xml_data to cachefile
writeCachedData(os.sep + "output" + os.sep + cacheKeyFile, xml_data, 0)
except spb.makiExceptions.MakiUseCache:
if(DEBUGLEVEL > 0): debug("Can use the cached process data!")
lastGoodFilename = os.sep + "output" + os.sep + cacheKeyFile
canUseCache = 1
xml_data = None
except:
# the old cache file is no good, so remove it
oldCacheFilename = os.sep + "output" + os.sep + cacheKeyFile
removeCachedData(oldCacheFilename)
raise
outbuffer.close()
if(DEBUGLEVEL > 0): debug("TIME FOR PROCESS STEP: "+str(time.time() - steptimein) + " seconds")
else:
raise RuntimeError, "Unrecognized step type: "+step.stepType
def replaceAliases(s):
if(not s): return s
if("$" not in s): return s
in_parts = s.split("/")
out_parts = []
for part in in_parts:
if(part):
if(part[0] == "$"):
val = aliases.get(part[1:], None)
if(val != None):
out_parts.append(val)
continue
out_parts.append(part)
return string.join(out_parts, "/")
def getRuleFromConfig(attrs):
match = replaceAliases(attrs.get((None,'match'), ''))
if(not match): raise RuntimeError, "encountered <rule> without @match"
if(DEBUGLEVEL > 1): debug("working on config for <rule match='%s'>" % match)
matchType = attrs.get((None,'matchType'), '')
if(matchType != 're'): matchType = 'fn'
return Rule(matchType, match)
def getStepFromConfig(name, attrs):
stepAttributes = {}
for key in attrs.keys():
if(key[1] == "file"):
stepAttributes[key[1]] = replaceAliases(attrs[key])
else:
stepAttributes[key[1]] = attrs[key]
return Step(name, stepAttributes)
class ZoneHandler(xml.sax.saxlib.ContentHandler):
def __init__(self, zone):
self.zone = zone
self.zone.clearRules()
self.stack = []
self.chars = ''
self.rule = None
def characters(self, content):
if(self.stack): self.chars += content
def startElementNS(self, name, qname, attrs):
global aliases
localname = name[1]
self.stack.append(localname)
fullname = '/' + string.join(self.stack, '/')
if(fullname not in KNOWN_ZONE_ELEMENTS):
raise RuntimeError, "Unknown element encountered in zone config file %s: '%s'" % (self.zone.configfile, fullname)
if(localname == 'rule'):
self.rule = getRuleFromConfig(attrs)
elif(localname in ('stylesheet', 'process', 'write', 'debug')):
self.rule.addStep(getStepFromConfig(localname, attrs))
def endElementNS(self, name, qname):
val = string.strip(self.chars)
self.stack.pop()
self.chars = ''
localname = name[1]
if(localname == 'rule'):
self.zone.addRule(self.rule)
class ConfigHandler(xml.sax.saxlib.ContentHandler):
def __init__(self):
self.stack = []
self.chars = ''
self.rule = None
def characters(self, content):
if(self.stack): self.chars += content
def startElementNS(self, name, qname, attrs):
global aliases
global zones
localname = name[1]
self.stack.append(localname)
fullname = '/' + string.join(self.stack, '/')
if(fullname not in KNOWN_CONFIG_ELEMENTS):
raise RuntimeError, "Unknown element encountered in config file: '%s'" % fullname
if(localname == 'alias'):
name = attrs.get((None, 'name'), '')
if(not name):
warn("encountered an <alias> without @name")
return
value = replaceAliases(attrs.get((None,'value'), ''))
if(not value):
warn("encountered an <alias> without @value")
return
if(DEBUGLEVEL > 1): debug("adding alias for '%s' -> '%s'" % (name, value))
aliases[name] = value
elif(localname == 'zone'):
match = replaceAliases(attrs.get((None,'match'), ''))
if(not match):
warn("encountered a <zone> without @match")
return
filename = replaceAliases(attrs.get((None,'file'), ''))
if(not filename):
warn("encountered a <zone> without @file")
return
if(DEBUGLEVEL > 1): debug("adding Zone for <zone match='%s' file='%s'>" % (match, filename))
zones.append(Zone(match, filename))
elif(localname == 'rule'):
self.rule = getRuleFromConfig(attrs)
elif(localname in ('stylesheet', 'process', 'write', 'debug')):
self.rule.addStep(getStepFromConfig(localname, attrs))
def endElementNS(self, name, qname):
global DEBUGLEVEL
global defaultTransformer
global fancyErrorPage
val = string.strip(self.chars)
self.stack.pop()
self.chars = ''
localname = name[1]
if(localname == 'debug-level'):
DEBUGLEVEL = int(val)
if(DEBUGLEVEL > 0): debug("set DEBUGLEVEL to %d" % DEBUGLEVEL)
elif(localname == 'fancy-error-page'):
if(val == "yes"): fancyErrorPage = 1
elif(val == "no"): fancyErrorPage = 0
else: warn("unrecognized value for <fancy-error-page>")
if(DEBUGLEVEL > 0): debug("set fancyErrorPage to %s" % val)
elif(localname == 'default-transformer'):
if(val in KNOWN_TRANSFORMERS):
if(DEBUGLEVEL > 0): debug("set defaultTransformer to "+val)
defaultTransformer = val
else:
warn("unrecognized value for <default-transformer>")
elif(localname == 'rule'):
zones[0].addRule(self.rule)
def endDocument(self):
zones.reverse()
def getConfig():
global zones
global configPath
global configLastRead
global DEBUGLEVEL
global aliases
global defaultTransformer
if(not os.path.exists(configPath)):
raise RuntimeError, "Config file does not exist: "+configPath
file_mtime = getFileMtime(configPath)
if(configLastRead >= file_mtime):
if(DEBUGLEVEL > 1): debug("using cached zone/rule config")
return
if(DEBUGLEVEL > 1): debug("about to try to load config data...")
data = getFile(configPath)
zones = []
zones.append(Zone('', '')) # add the default zone
aliases = {}
handler = ConfigHandler()
spb.XmlUtil.parseString(data, handler, 1)
if(DEBUGLEVEL > 1): debug("successfully loaded config data")
configLastRead = file_mtime
if(not defaultTransformer): raise RuntimeError, "Configuration error: no default-transformer specified."
def showTotalTime(timein):
if(DEBUGLEVEL > 0):
timeout = time.time()
debug("TOTAL PROCESSING TIME: "+str(timeout - timein) + " seconds")
def handleTransaction(trans):
"""this is the main entry point into this module.
'trans' is an instance of a subclass of genericHttpTransaction.
"""
global zones
global fancyErrorPage
timein = time.time()
if(DEBUGLEVEL > 1): debug("entered handler()")
try:
getConfig()
process(trans, timein)
showTotalTime(timein)
except:
etype = sys.exc_info()[0]
e = sys.exc_info()[1]
if(isinstance(e, spb.makiExceptions.MakiReturn)):
showTotalTime(timein)
else:
error(e)
if(fancyErrorPage):
htmlErrorPage(trans, e)
else:
#trans.setHeader("Status", 500)
#trans.sendHeaders()
pass
|