BuildDocs.py :  » XML » 4Suite » 4Suite-XML-1.0.2 » Ft » Lib » DistExt » Python Open Source

Home
Python Open Source
1.3.1.2 Python
2.Ajax
3.Aspect Oriented
4.Blog
5.Build
6.Business Application
7.Chart Report
8.Content Management Systems
9.Cryptographic
10.Database
11.Development
12.Editor
13.Email
14.ERP
15.Game 2D 3D
16.GIS
17.GUI
18.IDE
19.Installer
20.IRC
21.Issue Tracker
22.Language Interface
23.Log
24.Math
25.Media Sound Audio
26.Mobile
27.Network
28.Parser
29.PDF
30.Project Management
31.RSS
32.Search
33.Security
34.Template Engines
35.Test
36.UML
37.USB Serial
38.Web Frameworks
39.Web Server
40.Web Services
41.Web Unit
42.Wiki
43.Windows
44.XML
Python Open Source » XML » 4Suite 
4Suite » 4Suite XML 1.0.2 » Ft » Lib » DistExt » BuildDocs.py
########################################################################
# $Header: /var/local/cvsroot/4Suite/Ft/Lib/DistExt/BuildDocs.py,v 1.49.2.4 2006/11/25 23:28:28 jkloth Exp $
"""
Main distutils extensions for generating documentation

Copyright 2005 Fourthought, Inc. (USA).
Detailed license and copyright information: http://4suite.org/COPYRIGHT
Project home, documentation, distributions: http://4suite.org/
"""

# NOTE: Before anybody gets wild ideas about changing urllib stuff to
# Ft.Lib.Uri, remember that this code is used *BEFORE* Ft is installed.
# It also is "safe" to urllib stuff as it will only be dealing with local
# files and its filename <-> url conversion works within itself.

import sys, os, copy, warnings, re, imp, rfc822, time, cStringIO
from distutils import util
from distutils.core import Command,DEBUG
from distutils.errors import *

from Ft import GetConfigVar
from Ft.Lib import Uri,ImportUtil
from Ft.Lib.DistExt import Structures
from Ft.Lib.DistExt.Formatters import *

__zipsafe__ = True

class BuildDocs(Command):

    command_name = 'build_docs'

    description = "build documentation files (copy or generate XML sources)"

    user_options = [
        ('build-dir=', 'd',
         "directory to \"build\" (generate) to"),
        ('force', 'f',
         "forcibly build everything (ignore file timestamps)"),
        ]

    boolean_options = ['inplace', 'force']

    def initialize_options(self):
        self.build_dir = None
        self.build_lib = None
        self.force = None

        # If 'inplace' is True, the generated documents are considered
        # to be source files as well.
        self.inplace = False

        # Set to true to validate the static XML documents
        self.validate = False
        return

    def finalize_options(self):
        self.set_undefined_options('build',
                                   ('build_docs', 'build_dir'),
                                   ('build_lib', 'build_lib'),
                                   ('force', 'force'))

        # Get the distribution options that are used for build_docs
        # options.
        self.files = [ doc for doc in self.distribution.doc_files
                       if isinstance(doc, Structures.File) ]

        self.static = [ doc for doc in self.distribution.doc_files
                        if isinstance(doc, Structures.Document) ]

        self.modules, self.module_info = self.get_modules()

        self.extensions = [ doc for doc in self.distribution.doc_files
                            if isinstance(doc, Structures.ExtensionsDocument) ]

        self.scripts = [ doc for doc in self.distribution.scripts
                         if isinstance(doc, Structures.Script)
                         and doc.application is not None ]

        # Initialize the properties that are not externally modifiable.
        self._xslt_processor = None
        return

    def get_outputs(self):
        outputs = []
        for name in self.modules:
            xmlfile = self.get_output_filename(name, 'modules')
            outputs.append(xmlfile)
        for ext in self.extensions:
            xmlfile = self.get_output_filename(ext.name, 'extensions')
            outputs.append(xmlfile)
        for script in self.scripts:
            xmlfile = self.get_output_filename(script.name, 'commandline')
            outputs.append(xmlfile)
        index_name = 'index-' + self.distribution.get_name()
        outputs.append(self.get_output_filename(index_name))
        return outputs

    def get_source_files(self):
        sources = []
        for doc in self.distribution.doc_files:
            if isinstance(doc, Structures.File):
                source = util.convert_path(doc.source)
                sources.append(source)
            elif isinstance(doc, Structures.Document):
                source = util.convert_path(doc.source)
                sources.append(source)
                prefix = len(os.getcwd()) + len(os.sep)
                for path in self.find_xml_includes(Uri.OsPathToUri(source)):
                    sources.append(path[prefix:])
        if self.inplace:
            sources.extend(self.get_outputs())
        return sources

    def find_xml_includes(self, uri, _includes=None):
        if _includes is None:
            _includes = {}
        def gather_includes(fullurl):
            if fullurl not in _includes:
                _includes[fullurl] = Uri.UriToOsPath(fullurl)
                self.find_xml_includes(fullurl, _includes)
            return
        ProcessIncludes(uri, gather_includes)
        return _includes.values()

    def run(self):
        if self.modules:
            self.prepare_modules()

        documents = []
        # Add the static XML content
        if self.static:
            documents.extend(self.build_static())

        # Create the XML content for the API reference
        if self.modules:
            documents.extend(self.build_api())

        # Create the XML content for XPath/XSLT extensions
        if self.extensions:
            documents.extend(self.build_extensions())

        # Create the XML content for command-line applications
        if self.scripts:
            documents.extend(self.build_commandline())

        # Create the XML content for the index
        self.build_index(documents)
        return

    def get_modules(self):
        # Locate all installed modules
        modules = []
        sources = {}
        if self.distribution.has_pure_modules():
            build_py = self.get_finalized_command('build_py')
            for package, module, filename in build_py.find_all_modules():
                if module == '__init__':
                    module = package
                    module_type = imp.PKG_DIRECTORY
                elif package:
                    module = package + '.' + module
                    module_type = imp.PY_SOURCE
                modules.append(module)
                package_dir = package.replace('.', os.sep)
                package_dir = os.path.join(build_py.build_lib, package_dir)
                filename = os.path.basename(filename)
                filename = os.path.join(package_dir, filename)
                sources[module] = (filename, module_type)
        if self.distribution.has_ext_modules():
            build_ext = self.get_finalized_command('build_ext')
            for ext in build_ext.extensions:
                module = build_ext.get_ext_fullname(ext.name)
                filename = build_ext.get_ext_filename(module)
                modules.append(module)
                filename = os.path.join(build_ext.build_lib, filename)
                sources[module] = (filename, imp.C_EXTENSION)
        return (modules, sources)

    # -- Top-level worker functions ------------------------------------
    # (called from 'run()')

    def prepare_modules(self):
        if self.distribution.has_pure_modules():
            self.run_command('build_py')
        if self.distribution.has_ext_modules():
            self.run_command('build_ext')

        if self.distribution.config_module:
            from Ft.Lib.DistExt.InstallConfig import METADATA_KEYS
            if self.distribution.config_module not in sys.modules:
                module = imp.new_module(self.distribution.config_module)
                sys.modules[self.distribution.config_module] = module
            else:
                module = sys.modules[self.distribution.config_module]
            for name in METADATA_KEYS:
                value = getattr(self.distribution, 'get_' + name)()
                setattr(module, name.upper(), value)

        # Add the build directory to the search path (sys.path).
        sys.path.insert(0, self.build_lib)

        # Enable importing of modules in namespace packages.
        for package in self.distribution.namespace_packages:
            packages = [package]
            while '.' in package:
                package = '.'.join(package.split('.')[:-1])
                packages.insert(0, package)
            for package in packages:
                path = os.path.join(self.build_lib, *package.split('.'))
                if package not in sys.modules:
                    module = sys.modules[package] = imp.new_module(package)
                    module.__path__ = [path]
                else:
                    module = sys.modules[package]
                    try:
                        search_path = module.__path__
                    except AttributeError:
                        raise DistutilsSetupError("namespace package '%s' is"
                                                  " not a package" % package)
                    search_path.insert(0, path)

        # Any packages that are already imported also need their
        # search path (__path__) adjusted.
        for name in self.modules:
            if name in sys.modules:
                search_path = getattr(sys.modules[name], '__path__', None)
                if search_path is not None:
                    path = os.path.join(self.build_lib, *name.split('.'))
                    search_path.insert(0, path)
        return

    def build_static(self):
        documents = []
        for document in self.static:
            # Building is as simple as creating a new Document instance to
            # reflect the converted paths.
            document = copy.copy(document)
            document.source = util.convert_path(document.source)
            documents.append(document)

        # Validate the content
        if self.validate:
            from xml.sax.handler import feature_validation
            from Ft.Xml.InputSource import DefaultFactory
            from Ft.Xml.Sax import CreateParser

            class ErrorHandler:
                def __init__(self, displayhook):
                    self.displayhook = displayhook
                def warning(self, exception):
                    self.displayhook(exception)
                def error(self, exception):
                    self.displayhook(exception)
                def fatalError(self, exception):
                    raise exception

            parser = CreateParser()
            parser.setFeature(feature_validation, True)
            parser.setErrorHandler(ErrorHandler(self.warn))

            for document in documents:
                self.announce('validating %s' % document.source, 2)
                parser.parse(DefaultFactory.fromUri(document.source))
        return documents

    def build_api(self):
        formatter = ApiFormatter.ApiFormatter(self, self.module_info)
        category = 'modules'

        # Find the top-level package to be documented
        first = min(self.modules)
        last = max(self.modules)
        shortest = min(len(first), len(last))
        for i in xrange(shortest):
            if first[i] != last[i]:
                top_level = first[:i]
                break
        else:
            top_level = first[:shortest]
        if not top_level:
            raise DistutilsInternalError(
                "documenting multiple top-level packages is not supported")
        warnings.filterwarnings('ignore', '', DeprecationWarning, top_level)

        documents = []
        for name in self.modules:
            try:
                module = __import__(name, {}, {}, ['*'])
            except ImportError, error:
                if DEBUG: raise
                self.announce('not documenting %s (%s)' % (name, error), 3)
                continue

            # The build tree source is required for C-extension modules and
            # is safe for pure-Python modules.
            sources = [self.module_info[name][0]]
            xmlfile = self.document(category, name, sources, module, formatter)
            if name == top_level:
                # Only those documents with a title will be listed on the
                # index page
                title = '%s API Reference' % self.distribution.get_name()
                documents.append(Structures.Document(xmlfile,
                                                     stylesheet=category,
                                                     title=title,
                                                     category='general'))
        return documents

    def build_extensions(self):
        """
        Create XML documentation for XPath/XSLT extensions
        """
        formatter = ExtensionFormatter.ExtensionFormatter(self)
        category = 'extensions'

        extension_attrs = ('ExtNamespaces', 'ExtFunctions', 'ExtElements')

        docs = []
        for extension in self.extensions:
            # create a temporary module that will contain the combined
            # extension information
            extension_module = imp.new_module(extension.name)
            for attr in extension_attrs:
                setattr(extension_module, attr, {})
            sources = []
            for name in extension.modules:
                try:
                    module = __import__(name, {}, {}, extension_attrs)
                except ImportError, e:
                    raise DistutilsFileError(
                        "could not import '%s': %s" % (name, e))
                for attr in extension_attrs:
                    if hasattr(module, attr):
                        attrs = getattr(module, attr)
                        getattr(extension_module, attr).update(attrs)
                sources.append(self.module_info[name][0])
            xmlfile = self.document(category, extension.name, sources,
                                    extension_module, formatter)
            docs.append(Structures.Document(xmlfile, stylesheet=category,
                                            title=extension.title,
                                            category=category))
        return docs

    def build_commandline(self):
        formatter = CommandLineFormatter.CommandLineFormatter(self)
        category = 'commandline'

        docs = []
        for script in self.scripts:
            try:
                module = __import__(script.module, {}, {}, [script.application])
            except ImportError, e:
                raise DistutilsFileError(
                    "could not document '%s' script: %s" % (script.name, e))

            # Get the CommandLineApp instance for documenting
            app = getattr(module, script.application)()
            # Get the sources that are used to implement the application
            sources = [self.module_info[script.module][0]]
            for cmd_name, cmd in app.get_help_doc_info():
                source = cmd._fileName
                if source is None:
                    module_name = cmd.__class__.__module__
                    source = self.module_info[module_name][0]
                sources.append(source)
            # Now document the application and its commands, if any.
            xmlfile = self.document(category, script.name, sources, app,
                                    formatter)
            title = script.name + ' - ' +  app.description
            docs.append(Structures.Document(xmlfile, stylesheet=category,
                                            title=title, category=category))
        return docs

    def build_index(self, documents):
        from Ft.Xml.Xslt.BuiltInExtElements import RESERVED_NAMESPACE

        name = 'index-' + self.distribution.get_name()
        xmlfile = self.get_output_filename(name)
        source_mtime = max(os.path.getmtime(self.distribution.script_name),
                           os.path.getmtime(self.distribution.package_file),
                           ImportUtil.GetLastModified(__name__))
        try:
            target_mtime = os.path.getmtime(xmlfile)
        except OSError:
            target_mtime = -1
        if not (self.force or source_mtime > target_mtime):
            self.announce('not creating index (up-to-date)', 1)
            return
        else:
            self.announce("creating index -> %s" % xmlfile, 2)

        index = {}
        index_uri = Uri.OsPathToUri(xmlfile)
        xmlstr = XmlFormatter.XmlRepr().escape
        for doc in documents:
            if 'noindex' not in doc.flags:
                output = os.path.splitext(doc.source)[0] + '.html'
                source_uri = Uri.OsPathToUri(doc.source)
                output_uri = Uri.OsPathToUri(output)
                category = index.setdefault(doc.category, [])
                category.append({
                    'title' : xmlstr(doc.title),
                    'source' : Uri.Relativize(source_uri, index_uri),
                    'output' : Uri.Relativize(output_uri, index_uri),
                    'stylesheet' : xmlstr(doc.stylesheet),
                    })

        sections = []
        for title, category, sort in (
            ('General', 'general', False),
            ('Modules', 'modules', True),
            ('XPath/XSLT Extensions', 'extensions', False),
            ('Command-line Applications', 'commandline', True)
            ):
            if category not in index:
                continue
            items = []
            L = index[category]
            if sort:
                L.sort(lambda a, b: cmp(a['title'], b['title']))
            for info in L:
                repl = {'title' : info['title'],
                        'url' : info['output'],
                        }
                items.append(INDEX_LISTITEM % repl)
            if items:
                # add the section if it contains any entries
                items = ''.join(items)
                repl = {'title' : xmlstr(title),
                        'category' : xmlstr(category),
                        'items' : items,
                        }
                sections.append(INDEX_SECTION % repl)
        sections = ''.join(sections)

        sources = []
        for category in index.values():
            for info in category:
                sources.append(INDEX_SOURCE % info)
        sources = ''.join(sources)

        repl = {'fullname' : xmlstr(self.distribution.get_fullname()),
                'sections' : sections,
                'namespace' : RESERVED_NAMESPACE,
                'sources' : sources,
                }
        index = INDEX_TEMPLATE % repl

        if not self.dry_run:
            f = open(xmlfile, 'wb')
            f.write(index)
            f.close()

        return documents

    def document(self, category, name, sources, object, formatter):
        xmlfile = self.get_output_filename(name, category)
        self.mkpath(os.path.dirname(xmlfile))

        # The dependencies for 'object' are the source for the formatter
        # and, of course, 'sources'.
        formatter_module = formatter.__class__.__module__
        source_mtime = max(ImportUtil.GetLastModified(formatter_module),
                           *map(os.path.getmtime, sources))
        try:
            target_mtime = os.path.getmtime(xmlfile)
        except OSError:
            target_mtime = -1
        if self.force or source_mtime > target_mtime:
            self.announce("documenting %s -> %s" % (name, xmlfile), 2)
            if not self.dry_run:
                try:
                    stream = open(xmlfile, 'w')
                    try:
                        formatter.format(object, stream, encoding='iso-8859-1')
                    finally:
                        stream.close()
                except (KeyboardInterrupt, SystemExit):
                    os.remove(xmlfile)
                    raise
                except Exception, exc:
                    os.remove(xmlfile)
                    if DEBUG: raise
                    raise DistutilsExecError("could not document %s (%s)" %
                                             (name, exc))
        else:
            self.announce('not documenting %s (up-to-date)' % name, 1)
        return xmlfile

    def get_output_filename(self, name, category=None):
        dest_dir = self.build_dir
        if category:
            dest_dir = os.path.join(dest_dir, category)
        return os.path.join(dest_dir, name + '.xml')


def FindIncludes(source_uri, _includes=None):
    if _includes is None:
        _includes = {}
    def gather_includes(fullurl):
        if fullurl not in _includes:
            _includes[fullurl] = True
            FindIncludes(fullurl, _includes)
        return
    ProcessIncludes(source, gather_includes)
    return _includes


def ProcessIncludes(source, callback, xslt=False):
    from xml.sax import make_parser,SAXException,SAXNotRecognizedException
    from xml.sax.handler import ContentHandler,feature_namespaces,\
         feature_validation, feature_external_ges, feature_external_pes
    from xml.sax.xmlreader import InputSource

    # defined as nested to keep things "import clean"
    class InclusionFilter(ContentHandler):

        XSLT_INCLUDES = [
            ("http://www.w3.org/1999/XSL/Transform", "import"),
            ("http://www.w3.org/1999/XSL/Transform", "include"),
            ]

        def startDocument(self):
            url = self._locator.getSystemId()
            self._bases = [url]
            self._scheme = Uri.GetScheme(url)
            self._elements = [
                ("http://www.w3.org/2001/XInclude", "include"),
                ]
            if xslt:
                self._elements.extend(self.XSLT_INCLUDES)
        def startElementNS(self, expandedName, tagName, attrs):
            # Update xml:base stack
            xml_base = ("http://www.w3.org/XML/1998/namespace", "base")
            baseUri = attrs.get(xml_base, self._bases[-1])
            self._bases.append(baseUri)

            if expandedName in self._elements:
                try:
                    href = attrs[(None, 'href')]
                except KeyError:
                    # XInclude same document reference, nothing to do
                    return

                # Ignore XInclude's with parse='text'
                if attrs.get((None, 'parse'), 'xml') == 'text':
                    return

                # Only follow inclusions that have the same scheme as the
                # initial document.
                fullurl = Uri.BaseJoin(baseUri, href)
                if Uri.GetScheme(fullurl) == self._scheme:
                    callback(fullurl)
        def endElementNS(self, expandedName, tagName):
            del self._bases[-1]
    # -- end InclusionFilter --

    try:
        parser = make_parser()
        parser.setFeature(feature_namespaces, True)
        # Attempt to disable all external entity resolution
        for feature in (feature_validation, feature_external_ges,
                        feature_external_pes):
            try:
                parser.setFeature(feature, False)
            except SAXNotRecognizedException:
                pass
    except SAXException, e:
        raise DistutilsModuleError(e.getMessage())

    handler = InclusionFilter()
    parser.setContentHandler(handler)

    if isinstance(source, (str, unicode)):
        try:
            stream = Uri.UrlOpen(source)
        except OSError:
            # Assume part of an XInclude w/fallback.
            return
        source = InputSource(source)
        source.setByteStream(stream)
    elif hasattr(source, 'read'):
        stream = source
        source = InputSource(getattr(stream, 'name', None))
        source.setByteStream(stream)
    parser.parse(source)
    return


INDEX_TEMPLATE = """<?xml version="1.0" encoding="ISO-8859-1"?>
<!DOCTYPE article PUBLIC "-//OASIS//DTD Simplified DocBook XML V1.1//EN"
          "http://docbook.org/xml/simple/1.1/sdocbook.dtd">
<?ftdb-ignore-namespace http://xmlns.4suite.org/reserved?>
<article>
  <title>%(fullname)s Document Index</title>
%(sections)s

  <f:sources xmlns:f="%(namespace)s">
%(sources)s
  </f:sources>

</article>
"""

INDEX_SECTION = """
  <section id="%(category)s">
    <title>%(title)s</title>
    <itemizedlist>
%(items)s
    </itemizedlist>
  </section> <!-- %(category)s -->
"""

INDEX_LISTITEM = """\
      <listitem>
        <ulink url="%(url)s" type="generate">%(title)s</ulink>
      </listitem>
"""

INDEX_SOURCE = """\
    <f:source>
      <f:title>%(title)s</f:title>
      <f:src>%(source)s</f:src>
      <f:dst>%(output)s</f:dst>
      <f:stylesheet>%(stylesheet)s</f:stylesheet>
    </f:source>
"""
www.java2java.com | Contact Us
Copyright 2009 - 12 Demo Source and Support. All rights reserved.
All other trademarks are property of their respective owners.