Docs generation files for lsp-status.nvim (modified from core Neovim files)

gen_vimdoc.py

#!/usr/bin/env python3
"""Generates Nvim :help docs from C/Lua docstrings, using Doxygen.

Flow:
    main
      extract_from_xml
        fmt_node_as_vimhelp   \
          para_as_map          } recursive
            update_params_map /
              render_node

This would be easier using lxml and XSLT, but:

  1. This should avoid needing Python dependencies, especially ones that are
     C modules that have library dependencies (lxml requires libxml and
     libxslt).
  2. I wouldn't know how to deal with nested indentation in <para> tags using
     XSLT.

Each function :help block is formatted as follows:

  - Max width of 78 columns (`text_width`).
  - Indent with spaces (not tabs).
  - Indent of 16 columns for body text.
  - Function signature and helptag (right-aligned) on the same line.
    - Signature and helptag must have a minimum of 8 spaces between them.
    - If the signature is too long, it is placed on the line after the helptag.
      Signature wraps at `text_width - 8` characters with subsequent
      lines indented to the open parenthesis.
    - Subsection bodies are indented an additional 4 spaces.
  - Body consists of function description, parameters, return description, and
    C declaration (`INCLUDE_C_DECL`).
  - Parameters are omitted for the `void` and `Error *` types, or if the
    parameter is marked as [out].
  - Each function documentation is separated by a single line.
"""
import os
import re
import sys
import shutil
import textwrap
import subprocess
import collections

from xml.dom import minidom

if sys.version_info[0] < 3 or sys.version_info[1] < 5:
    print("requires Python 3.5+")
    sys.exit(1)

DEBUG = ('DEBUG' in os.environ)
TARGET = os.environ.get('TARGET', None)
INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ)
INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ)

fmt_vimhelp = False  # HACK
text_width = 78
script_path = os.path.abspath(__file__)
base_dir = os.path.dirname(script_path)
out_dir = os.path.join(base_dir, 'tmp-{target}-doc')
filter_cmd = '%s %s' % (sys.executable, script_path)
seen_funcs = set()
lua2dox_filter = os.path.join(base_dir, 'lua2dox_filter')

CONFIG = {
    'lsp-status': {
        'mode': 'lua',
        'filename': 'lsp-status.txt',
        'section_start_token': '*lsp-status-api*',
        'section_order': [
          'lsp-status.lua',
          'clangd.lua',
          'pyls_ms.lua',
        ],
        'files': ' '.join([
            os.path.join(base_dir, 'lua/lsp-status.lua'),
            os.path.join(base_dir, 'lua/lsp-status/extensions'),
        ]),
        'file_patterns': '*.lua',
        'fn_name_prefix': '',
        'section_name': {
          'lsp-status': 'api'
        },
        'section_fmt': lambda name: (
          'API: core module lsp-status'
          if name.lower() == 'lsp-status'
          else f'Module: lsp-status.{name.lower()}'),
        'helptag_fmt': lambda name: (
          '*lsp-status-api*'
          if name.lower() == 'lsp-status'
          else f'*lsp-status-api-{name.lower()}*'),
        'fn_helptag_fmt': lambda fstem, name: (
          f'*{fstem}.{name}()*'
          if fstem.lower() == 'lsp-status'
          else f'*lsp-status.{fstem}.{name}()*'),
        'module_override': {
          'clangd': 'extensions.clangd',
          'pyls_ms': 'extensions.pyls_ms',
        },
        'append_only': [],
    },
}

param_exclude = ()

# Annotations are displayed as line items after API function descriptions.
annotation_map = {
    'FUNC_API_FAST': '{fast}',
}


# Tracks `xrefsect` titles.  As of this writing, used only for separating
# deprecated functions.
xrefs = set()


# Raises an error with details about `o`, if `cond` is in object `o`,
# or if `cond()` is callable and returns True.
def debug_this(cond, o):
    name = ''
    if not isinstance(o, str):
        try:
            name = o.nodeName
            o = o.toprettyxml(indent='  ', newl='\n')
        except Exception:
            pass
    if ((callable(cond) and cond())
            or (not callable(cond) and cond)
            or (not callable(cond) and cond in o)):
        raise RuntimeError('xxx: {}\n{}'.format(name, o))


def find_first(parent, name):
    """Finds the first matching node within parent."""
    sub = parent.getElementsByTagName(name)
    if not sub:
        return None
    return sub[0]


def iter_children(parent, name):
    """Yields matching child nodes within parent."""
    for child in parent.childNodes:
        if child.nodeType == child.ELEMENT_NODE and child.nodeName == name:
            yield child


def get_child(parent, name):
    """Gets the first matching child node."""
    for child in iter_children(parent, name):
        return child
    return None


def self_or_child(n):
    """Gets the first child node, or self."""
    if len(n.childNodes) == 0:
        return n
    return n.childNodes[0]


def clean_text(text):
    """Cleans text.

    Only cleans superfluous whitespace at the moment.
    """
    return ' '.join(text.split()).strip()


def clean_lines(text):
    """Removes superfluous lines.

    The beginning and end of the string is trimmed.  Empty lines are collapsed.
    """
    return re.sub(r'\A\n\s*\n*|\n\s*\n*\Z', '', re.sub(r'(\n\s*\n+)+', '\n\n', text))


def is_blank(text):
    return '' == clean_lines(text)


def get_text(n, preformatted=False):
    """Recursively concatenates all text in a node tree."""
    text = ''
    if n.nodeType == n.TEXT_NODE:
        return n.data
    if n.nodeName == 'computeroutput':
        for node in n.childNodes:
            text += get_text(node)
        return '`{}` '.format(text)
    for node in n.childNodes:
        if node.nodeType == node.TEXT_NODE:
            text += node.data if preformatted else clean_text(node.data)
        elif node.nodeType == node.ELEMENT_NODE:
            text += ' ' + get_text(node, preformatted)
    return text


# Gets the length of the last line in `text`, excluding newline ("\n") char.
def len_lastline(text):
    lastnl = text.rfind('\n')
    if -1 == lastnl:
        return len(text)
    if '\n' == text[-1]:
        return lastnl - (1 + text.rfind('\n', 0, lastnl))
    return len(text) - (1 + lastnl)


def len_lastline_withoutindent(text, indent):
    n = len_lastline(text)
    return (n - len(indent)) if n > len(indent) else 0


# Returns True if node `n` contains only inline (not block-level) elements.
def is_inline(n):
    # if len(n.childNodes) == 0:
    #     return n.nodeType == n.TEXT_NODE or n.nodeName == 'computeroutput'
    for c in n.childNodes:
        if c.nodeType != c.TEXT_NODE and c.nodeName != 'computeroutput':
            return False
        if not is_inline(c):
            return False
    return True


def doc_wrap(text, prefix='', width=70, func=False, indent=None):
    """Wraps text to `width`.

    First line is prefixed with `prefix`, subsequent lines are aligned.
    If `func` is True, only wrap at commas.
    """
    if not width:
        # return prefix + text
        return text

    # Whitespace used to indent all lines except the first line.
    indent = ' ' * len(prefix) if indent is None else indent
    indent_only = (prefix == '' and indent is not None)

    if func:
        lines = [prefix]
        for part in text.split(', '):
            if part[-1] not in ');':
                part += ', '
            if len(lines[-1]) + len(part) > width:
                lines.append(indent)
            lines[-1] += part
        return '\n'.join(x.rstrip() for x in lines).rstrip()

    # XXX: Dummy prefix to force TextWrapper() to wrap the first line.
    if indent_only:
        prefix = indent

    tw = textwrap.TextWrapper(break_long_words=False,
                              break_on_hyphens=False,
                              width=width,
                              initial_indent=prefix,
                              subsequent_indent=indent)
    result = '\n'.join(tw.wrap(text.strip()))

    # XXX: Remove the dummy prefix.
    if indent_only:
        result = result[len(indent):]

    return result


def max_name(names):
    if len(names) == 0:
        return 0
    return max(len(name) for name in names)


def update_params_map(parent, ret_map, width=62):
    """Updates `ret_map` with name:desc key-value pairs extracted
    from Doxygen XML node `parent`.
    """
    params = collections.OrderedDict()
    for node in parent.childNodes:
        if node.nodeType == node.TEXT_NODE:
            continue
        name_node = find_first(node, 'parametername')
        if name_node.getAttribute('direction') == 'out':
            continue
        name = get_text(name_node)
        if name in param_exclude:
            continue
        params[name.strip()] = node
    max_name_len = max_name(params.keys()) + 8
    # `ret_map` is a name:desc map.
    for name, node in params.items():
        desc = ''
        desc_node = get_child(node, 'parameterdescription')
        if desc_node:
            desc = fmt_node_as_vimhelp(
                    desc_node, width=width, indent=(' ' * max_name_len))
            ret_map[name] = desc
    return ret_map


def render_node(n, text, prefix='', indent='', width=62):
    """Renders a node as Vim help text, recursively traversing all descendants."""
    global fmt_vimhelp

    def ind(s):
        return s if fmt_vimhelp else ''

    text = ''
    # space_preceding = (len(text) > 0 and ' ' == text[-1][-1])
    # text += (int(not space_preceding) * ' ')

    if n.nodeName == 'preformatted':
        o = get_text(n, preformatted=True)
        ensure_nl = '' if o[-1] == '\n' else '\n'
        text += '>{}{}\n<'.format(ensure_nl, o)
    elif is_inline(n):
        text = doc_wrap(get_text(n), indent=indent, width=width)
    elif n.nodeName == 'verbatim':
        # TODO: currently we don't use this. The "[verbatim]" hint is there as
        # a reminder that we must decide how to format this if we do use it.
        text += ' [verbatim] {}'.format(get_text(n))
    elif n.nodeName == 'listitem':
        for c in n.childNodes:
            text += (
                indent
                + prefix
                + render_node(c, text, indent=indent + (' ' * len(prefix)), width=width)
            )
    elif n.nodeName in ('para', 'heading'):
        for c in n.childNodes:
            text += render_node(c, text, indent=indent, width=width)
    elif n.nodeName == 'itemizedlist':
        for c in n.childNodes:
            text += '{}\n'.format(render_node(c, text, prefix='• ',
                                              indent=indent, width=width))
    elif n.nodeName == 'orderedlist':
        i = 1
        for c in n.childNodes:
            if is_blank(get_text(c)):
                text += '\n'
                continue
            text += '{}\n'.format(render_node(c, text, prefix='{}. '.format(i),
                                              indent=indent, width=width))
            i = i + 1
    elif n.nodeName == 'simplesect' and 'note' == n.getAttribute('kind'):
        text += 'Note:\n    '
        for c in n.childNodes:
            text += render_node(c, text, indent='  ', width=width)
        text += '\n'
    elif n.nodeName == 'simplesect' and 'warning' == n.getAttribute('kind'):
        text += 'Warning:\n    '
        for c in n.childNodes:
            text += render_node(c, text, indent='  ', width=width)
        text += '\n'
    elif (n.nodeName == 'simplesect'
            and n.getAttribute('kind') in ('return', 'see')):
        text += ind('    ')
        for c in n.childNodes:
            text += render_node(c, text, indent='  ', width=width)
    else:
        raise RuntimeError('unhandled node type: {}\n{}'.format(
            n.nodeName, n.toprettyxml(indent='  ', newl='\n')))
    return text


def para_as_map(parent, indent='', width=62):
    """Extracts a Doxygen XML <para> node to a map.

    Keys:
        'text': Text from this <para> element
        'params': <parameterlist> map
        'return': List of @return strings
        'seealso': List of @see strings
        'xrefs': ?
    """
    chunks = {
        'text': '',
        'params': collections.OrderedDict(),
        'return': [],
        'seealso': [],
        'xrefs': []
    }

    # Ordered dict of ordered lists.
    groups = collections.OrderedDict([
        ('params', []),
        ('return', []),
        ('seealso', []),
        ('xrefs', []),
    ])

    # Gather nodes into groups.  Mostly this is because we want "parameterlist"
    # nodes to appear together.
    text = ''
    kind = ''
    last = ''
    if is_inline(parent):
        # Flatten inline text from a tree of non-block nodes.
        text = doc_wrap(render_node(parent, ""), indent=indent, width=width)
    else:
        prev = None  # Previous node
        for child in parent.childNodes:
            if child.nodeName == 'parameterlist':
                groups['params'].append(child)
            elif child.nodeName == 'xrefsect':
                groups['xrefs'].append(child)
            elif child.nodeName == 'simplesect':
                last = kind
                kind = child.getAttribute('kind')
                if kind == 'return' or (kind == 'note' and last == 'return'):
                    groups['return'].append(child)
                elif kind == 'see':
                    groups['seealso'].append(child)
                elif kind in ('note', 'warning'):
                    text += render_node(child, text, indent=indent, width=width)
                else:
                    raise RuntimeError('unhandled simplesect: {}\n{}'.format(
                        child.nodeName, child.toprettyxml(indent='  ', newl='\n')))
            else:
                if (prev is not None
                        and is_inline(self_or_child(prev))
                        and is_inline(self_or_child(child))
                        and '' != get_text(self_or_child(child)).strip()
                        and ' ' != text[-1]):
                    text += ' '
                text += render_node(child, text, indent=indent, width=width)
                prev = child

    chunks['text'] += text

    # Generate map from the gathered items.
    if len(groups['params']) > 0:
        for child in groups['params']:
            update_params_map(child, ret_map=chunks['params'], width=width)
    for child in groups['return']:
        chunks['return'].append(render_node(
            child, '', indent=indent, width=width))
    for child in groups['seealso']:
        chunks['seealso'].append(render_node(
            child, '', indent=indent, width=width))
    for child in groups['xrefs']:
        # XXX: Add a space (or any char) to `title` here, otherwise xrefs
        # ("Deprecated" section) acts very weird...
        title = get_text(get_child(child, 'xreftitle')) + ' '
        xrefs.add(title)
        xrefdesc = get_text(get_child(child, 'xrefdescription'))
        chunks['xrefs'].append(doc_wrap(xrefdesc, prefix='{}: '.format(title),
                                        width=width) + '\n')

    return chunks


def fmt_node_as_vimhelp(parent, width=62, indent=''):
    """Renders (nested) Doxygen <para> nodes as Vim :help text.

    NB: Blank lines in a docstring manifest as <para> tags.
    """
    rendered_blocks = []

    def fmt_param_doc(m):
        """Renders a params map as Vim :help text."""
        max_name_len = max_name(m.keys()) + 2
        out = ''
        for name, desc in m.items():
            name = '    {}'.format('{{{}}}'.format(name).ljust(max_name_len))
            out += '{}{}\n'.format(name, desc)
        return out.rstrip()

    def has_nonexcluded_params(m):
        """Returns true if any of the given params has at least
        one non-excluded item."""
        if fmt_param_doc(m) != '':
            return True

    for child in parent.childNodes:
        para = para_as_map(child, indent, width)

        # Generate text from the gathered items.
        chunks = [para['text']]
        if len(para['params']) > 0 and has_nonexcluded_params(para['params']):
            chunks.append('\nParameters: ~')
            chunks.append(fmt_param_doc(para['params']))
        if len(para['return']) > 0:
            chunks.append('\nReturn: ~')
            for s in para['return']:
                chunks.append(s)
        if len(para['seealso']) > 0:
            chunks.append('\nSee also: ~')
            for s in para['seealso']:
                chunks.append(s)
        for s in para['xrefs']:
            chunks.append(s)

        rendered_blocks.append(clean_lines('\n'.join(chunks).strip()))
        rendered_blocks.append('')
    return clean_lines('\n'.join(rendered_blocks).strip())


def extract_from_xml(filename, target, width):
    """Extracts Doxygen info as maps without formatting the text.

    Returns two maps:
      1. Functions
      2. Deprecated functions

    The `fmt_vimhelp` global controls some special cases for use by
    fmt_doxygen_xml_as_vimhelp(). (TODO: ugly :)
    """
    global xrefs
    global fmt_vimhelp
    xrefs.clear()
    fns = {}  # Map of func_name:docstring.
    deprecated_fns = {}  # Map of func_name:docstring.

    dom = minidom.parse(filename)
    compoundname = get_text(dom.getElementsByTagName('compoundname')[0])
    for member in dom.getElementsByTagName('memberdef'):
        if member.getAttribute('static') == 'yes' or \
                member.getAttribute('kind') != 'function' or \
                member.getAttribute('prot') == 'private' or \
                get_text(get_child(member, 'name')).startswith('_'):
            continue

        loc = find_first(member, 'location')
        if 'private' in loc.getAttribute('file'):
            continue

        return_type = get_text(get_child(member, 'type'))
        if return_type == '':
            continue

        if return_type.startswith(('ArrayOf', 'DictionaryOf')):
            parts = return_type.strip('_').split('_')
            return_type = '{}({})'.format(parts[0], ', '.join(parts[1:]))

        name = get_text(get_child(member, 'name'))

        annotations = get_text(get_child(member, 'argsstring'))
        if annotations and ')' in annotations:
            annotations = annotations.rsplit(')', 1)[-1].strip()
        # XXX: (doxygen 1.8.11) 'argsstring' only includes attributes of
        # non-void functions.  Special-case void functions here.
        if name == 'nvim_get_mode' and len(annotations) == 0:
            annotations += 'FUNC_API_FAST'
        annotations = filter(None, map(lambda x: annotation_map.get(x),
                                       annotations.split()))

        if not fmt_vimhelp:
            pass
        else:
            fstem = '?'
            if '.' in compoundname:
                fstem = compoundname.split('.')[0]
                fstem = CONFIG[target]['module_override'].get(fstem, fstem)
            vimtag = CONFIG[target]['fn_helptag_fmt'](fstem, name)

        params = []
        type_length = 0

        for param in iter_children(member, 'param'):
            param_type = get_text(get_child(param, 'type')).strip()
            param_name = ''
            declname = get_child(param, 'declname')
            if declname:
                param_name = get_text(declname).strip()
            elif CONFIG[target]['mode'] == 'lua':
                # XXX: this is what lua2dox gives us...
                param_name = param_type
                param_type = ''

            if param_name in param_exclude:
                continue

            if fmt_vimhelp and param_type.endswith('*'):
                param_type = param_type.strip('* ')
                param_name = '*' + param_name
            type_length = max(type_length, len(param_type))
            params.append((param_type, param_name))

        c_args = []
        for param_type, param_name in params:
            c_args.append(('    ' if fmt_vimhelp else '') + (
                '%s %s' % (param_type.ljust(type_length), param_name)).strip())

        prefix = '%s(' % name
        suffix = '%s)' % ', '.join('{%s}' % a[1] for a in params
                                   if a[0] not in ('void', 'Error'))
        if not fmt_vimhelp:
            c_decl = '%s %s(%s);' % (return_type, name, ', '.join(c_args))
            signature = prefix + suffix
        else:
            c_decl = textwrap.indent('%s %s(\n%s\n);' % (return_type, name,
                                                         ',\n'.join(c_args)),
                                     '    ')

            # Minimum 8 chars between signature and vimtag
            lhs = (width - 8) - len(vimtag)

            if len(prefix) + len(suffix) > lhs:
                signature = vimtag.rjust(width) + '\n'
                signature += doc_wrap(suffix, width=width-8, prefix=prefix,
                                      func=True)
            else:
                signature = prefix + suffix
                signature += vimtag.rjust(width - len(signature))

        paras = []
        desc = find_first(member, 'detaileddescription')
        if desc:
            for child in desc.childNodes:
                paras.append(para_as_map(child))
            if DEBUG:
                print(textwrap.indent(
                    re.sub(r'\n\s*\n+', '\n',
                           desc.toprettyxml(indent='  ', newl='\n')), ' ' * 16))

        fn = {
            'annotations': list(annotations),
            'signature': signature,
            'parameters': params,
            'parameters_doc': collections.OrderedDict(),
            'doc': [],
            'return': [],
            'seealso': [],
        }
        if fmt_vimhelp:
            fn['desc_node'] = desc  # HACK :(

        for m in paras:
            if 'text' in m:
                if not m['text'] == '':
                    fn['doc'].append(m['text'])
            if 'params' in m:
                # Merge OrderedDicts.
                fn['parameters_doc'].update(m['params'])
            if 'return' in m and len(m['return']) > 0:
                fn['return'] += m['return']
            if 'seealso' in m and len(m['seealso']) > 0:
                fn['seealso'] += m['seealso']

        if INCLUDE_C_DECL:
            fn['c_decl'] = c_decl

        if 'Deprecated' in str(xrefs):
            deprecated_fns[name] = fn
        elif name.startswith(CONFIG[target]['fn_name_prefix']):
            fns[name] = fn

        xrefs.clear()

    fns = collections.OrderedDict(sorted(fns.items()))
    deprecated_fns = collections.OrderedDict(sorted(deprecated_fns.items()))
    return (fns, deprecated_fns)


def fmt_doxygen_xml_as_vimhelp(filename, target):
    """Entrypoint for generating Vim :help from from Doxygen XML.

    Returns 3 items:
      1. Vim help text for functions found in `filename`.
      2. Vim help text for deprecated functions.
    """
    global fmt_vimhelp
    fmt_vimhelp = True
    fns_txt = {}  # Map of func_name:vim-help-text.
    deprecated_fns_txt = {}  # Map of func_name:vim-help-text.
    fns, _ = extract_from_xml(filename, target, width=text_width)

    for name, fn in fns.items():
        # Generate Vim :help for parameters.
        if fn['desc_node']:
            doc = fmt_node_as_vimhelp(fn['desc_node'])
        if not doc:
            doc = 'TODO: Documentation'

        annotations = '\n'.join(fn['annotations'])
        if annotations:
            annotations = ('\n\nAttributes: ~\n' +
                           textwrap.indent(annotations, '    '))
            i = doc.rfind('Parameters: ~')
            if i == -1:
                doc += annotations
            else:
                doc = doc[:i] + annotations + '\n\n' + doc[i:]

        if INCLUDE_C_DECL:
            doc += '\n\nC Declaration: ~\n>\n'
            doc += fn['c_decl']
            doc += '\n<'

        func_doc = fn['signature'] + '\n'
        func_doc += textwrap.indent(clean_lines(doc), ' ' * 16)
        func_doc = re.sub(r'^\s+([<>])$', r'\1', func_doc, flags=re.M)

        if 'Deprecated' in xrefs:
            deprecated_fns_txt[name] = func_doc
        elif name.startswith(CONFIG[target]['fn_name_prefix']):
            fns_txt[name] = func_doc

        xrefs.clear()

    fmt_vimhelp = False
    return ('\n\n'.join(list(fns_txt.values())),
            '\n\n'.join(list(deprecated_fns_txt.values())))


def delete_lines_below(filename, tokenstr):
    """Deletes all lines below the line containing `tokenstr`, the line itself,
    and one line above it.
    """
    lines = open(filename).readlines()
    i = 0
    found = False
    for i, line in enumerate(lines, 1):
        if tokenstr in line:
            found = True
            break
    if not found:
        raise RuntimeError(f'not found: "{tokenstr}"')
    i = max(0, i - 2)
    with open(filename, 'wt') as fp:
        fp.writelines(lines[0:i])


def main(config):
    """Generates:

    1. Vim :help docs

    Doxygen is called and configured through stdin.
    """
    for target in CONFIG:
        if TARGET is not None and target != TARGET:
            continue

        output_dir = out_dir.format(target=target)
        p = subprocess.Popen(
                ['doxygen', '-'],
                stdin=subprocess.PIPE,
                # silence warnings
                # runtime/lua/vim/lsp.lua:209: warning: argument 'foo' not found
                stderr=(subprocess.STDOUT if DEBUG else subprocess.DEVNULL))
        p.communicate(
            config.format(
                input=CONFIG[target]['files'],
                output=output_dir,
                filter=filter_cmd,
                file_patterns=CONFIG[target]['file_patterns'])
            .encode('utf8')
        )
        if p.returncode:
            sys.exit(p.returncode)

        sections = {}
        intros = {}
        sep = '=' * text_width

        base = os.path.join(output_dir, 'xml')
        dom = minidom.parse(os.path.join(base, 'index.xml'))

        # generate docs for section intros
        for compound in dom.getElementsByTagName('compound'):
            if compound.getAttribute('kind') != 'group':
                continue

            groupname = get_text(find_first(compound, 'name'))
            groupxml = os.path.join(base, '%s.xml' %
                                    compound.getAttribute('refid'))

            desc = find_first(minidom.parse(groupxml), 'detaileddescription')
            if desc:
                doc = fmt_node_as_vimhelp(desc)
                if doc:
                    intros[groupname] = doc
        for compound in dom.getElementsByTagName('compound'):
            if compound.getAttribute('kind') != 'file':
                continue

            filename = get_text(find_first(compound, 'name'))
            if filename.endswith('.c') or filename.endswith('.lua'):
                # Extract formatted (:help).
                functions_text, deprecated_text = fmt_doxygen_xml_as_vimhelp(
                    os.path.join(base, '{}.xml'.format(
                                 compound.getAttribute('refid'))), target)

                if not functions_text and not deprecated_text:
                    continue
                else:
                    name = os.path.splitext(
                            os.path.basename(filename))[0].lower()
                    sectname = name.upper() if name == 'ui' else name.title()
                    doc = ''
                    intro = intros.get(f'api-{name}')
                    if intro:
                        doc += '\n\n' + intro

                    if functions_text:
                        doc += '\n\n' + functions_text

                    if INCLUDE_DEPRECATED and deprecated_text:
                        doc += f'\n\n\nDeprecated {sectname} Functions: ~\n\n'
                        doc += deprecated_text

                    if doc:
                        filename = os.path.basename(filename)
                        sectname = CONFIG[target]['section_name'].get(
                                filename, sectname)
                        title = CONFIG[target]['section_fmt'](sectname)
                        helptag = CONFIG[target]['helptag_fmt'](sectname)
                        sections[filename] = (title, helptag, doc)

        assert sections
        if len(sections) > len(CONFIG[target]['section_order']):
            raise RuntimeError(
                'found new modules "{}"; update the "section_order" map'.format(
                    set(sections).difference(CONFIG[target]['section_order'])))

        docs = ''

        i = 0
        for filename in CONFIG[target]['section_order']:
            title, helptag, section_doc = sections.pop(filename)
            i += 1
            if filename not in CONFIG[target]['append_only']:
                docs += sep
                docs += '\n%s%s' % (title,
                                    helptag.rjust(text_width - len(title)))
            docs += section_doc
            docs += '\n\n\n'

        docs = docs.rstrip() + '\n\n'
        docs += ' vim:tw=78:ts=2:ft=help:norl:\n'

        doc_file = os.path.join(base_dir, 'doc',
                                CONFIG[target]['filename'])

        delete_lines_below(doc_file, CONFIG[target]['section_start_token'])
        with open(doc_file, 'ab') as fp:
            fp.write(docs.encode('utf8'))

        shutil.rmtree(output_dir)


def filter_source(filename):
    name, extension = os.path.splitext(filename)
    if extension == '.lua':
        p = subprocess.run([lua2dox_filter, filename], stdout=subprocess.PIPE)
        op = ('?' if 0 != p.returncode else p.stdout.decode('utf-8'))
        print(op)
    else:
        """Filters the source to fix macros that confuse Doxygen."""
        with open(filename, 'rt') as fp:
            print(re.sub(r'^(ArrayOf|DictionaryOf)(\(.*?\))',
                         lambda m: m.group(1)+'_'.join(
                             re.split(r'[^\w]+', m.group(2))),
                         fp.read(), flags=re.M))


Doxyfile = textwrap.dedent('''
    OUTPUT_DIRECTORY       = {output}
    INPUT                  = {input}
    INPUT_ENCODING         = UTF-8
    FILE_PATTERNS          = {file_patterns}
    RECURSIVE              = YES
    INPUT_FILTER           = "{filter}"
    EXCLUDE                =
    EXCLUDE_SYMLINKS       = NO
    EXCLUDE_PATTERNS       = */private/*
    EXCLUDE_SYMBOLS        =
    EXTENSION_MAPPING      = lua=C
    EXTRACT_PRIVATE        = NO
    EXTRACT_ALL            = NO

    GENERATE_HTML          = NO
    GENERATE_DOCSET        = NO
    GENERATE_HTMLHELP      = NO
    GENERATE_QHP           = NO
    GENERATE_TREEVIEW      = NO
    GENERATE_LATEX         = NO
    GENERATE_RTF           = NO
    GENERATE_MAN           = NO
    GENERATE_DOCBOOK       = NO
    GENERATE_AUTOGEN_DEF   = NO

    GENERATE_XML           = YES
    XML_OUTPUT             = xml
    XML_PROGRAMLISTING     = NO

    ENABLE_PREPROCESSING   = YES
    MACRO_EXPANSION        = YES
    EXPAND_ONLY_PREDEF     = NO
    MARKDOWN_SUPPORT       = YES
''')

if __name__ == "__main__":
    if len(sys.argv) > 1:
        filter_source(sys.argv[1])
    else:
        main(Doxyfile)

# vim: set ft=python ts=4 sw=4 tw=79 et :

initdoc.txt

*lsp-status.txt*        
*lsp-status.nvim*
*lsp-status.nvim*

Author: Wil Thomason <wil.thomason@gmail.com>

CONTENTS                                        *lsp-status-contents*
Introduction                                    |lsp-status-introduction|
API                                             |lsp-status-api|

==============================================================================
API                                              *lsp-status-api*

==============================================================================
CHANGELOG                                        *lsp-status-changelog*

==============================================================================
CONTRIBUTING                                     *lsp-status-contributing*

Issue reports, feature requests,  and pull requests with bug fixes or feature
additions are welcomed! Please use the tools for such at
https://github.com/wbthomason/lsp-status.nvim.

==============================================================================
vim:tw=78:ts=2:ft=help:norl

lua2dox.lua

--[[--------------------------------------------------------------------------
--   Copyright (C) 2012 by Simon Dales   --
--   simon@purrsoft.co.uk   --
--                                                                         --
--   This program is free software; you can redistribute it and/or modify  --
--   it under the terms of the GNU General Public License as published by  --
--   the Free Software Foundation; either version 2 of the License, or     --
--   (at your option) any later version.                                   --
--                                                                         --
--   This program is distributed in the hope that it will be useful,       --
--   but WITHOUT ANY WARRANTY; without even the implied warranty of        --
--   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         --
--   GNU General Public License for more details.                          --
--                                                                         --
--   You should have received a copy of the GNU General Public License     --
--   along with this program; if not, write to the                         --
--   Free Software Foundation, Inc.,                                       --
--   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             --
----------------------------------------------------------------------------]]

--[[!
Lua-to-Doxygen converter

Partially from lua2dox
http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm

Running
-------

This file "lua2dox.lua" gets called by "lua2dox_filter" (bash).

Doxygen must be on your system. You can experiment like so:

- Run "doxygen -g" to create a default Doxyfile.
- Then alter it to let it recognise lua. Add the two following lines:
    FILE_PATTERNS   = *.lua
    FILTER_PATTERNS = *.lua=lua2dox_filter
- Then run "doxygen".

The core function reads the input file (filename or stdin) and outputs some pseudo C-ish language.
It only has to be good enough for doxygen to see it as legal.

One limitation is that each line is treated separately (except for long comments).
The implication is that class and function declarations must be on the same line.
Some functions can have their parameter lists extended over multiple lines to make it look neat.
Managing this where there are also some comments is a bit more coding than I want to do at this stage,
so it will probably not document accurately if we do do this.

However I have put in a hack that will insert the "missing" close paren.
The effect is that you will get the function documented, but not with the parameter list you might expect.
]]

function class(BaseClass, ClassInitialiser)
  local newClass = {}    -- a new class newClass
  if not ClassInitialiser and type(BaseClass) == 'function' then
    ClassInitialiser = BaseClass
    BaseClass = nil
  elseif type(BaseClass) == 'table' then
    -- our new class is a shallow copy of the base class!
    for i,v in pairs(BaseClass) do
      newClass[i] = v
    end
    newClass._base = BaseClass
  end
  -- the class will be the metatable for all its newInstanceects,
  -- and they will look up their methods in it.
  newClass.__index = newClass

  -- expose a constructor which can be called by <classname>(<args>)
  local classMetatable = {}
  classMetatable.__call = 
  function(class_tbl, ...)
    local newInstance = {}
    setmetatable(newInstance,newClass)
    --if init then
    --	init(newInstance,...)
    if class_tbl.init then
      class_tbl.init(newInstance,...)
    else 
      -- make sure that any stuff from the base class is initialized!
      if BaseClass and BaseClass.init then
        BaseClass.init(newInstance, ...)
      end
    end
    return newInstance
  end
  newClass.init = ClassInitialiser
  newClass.is_a = 
  function(this, klass)
    local thisMetatable = getmetatable(this)
    while thisMetatable do 
      if thisMetatable == klass then
        return true
      end
      thisMetatable = thisMetatable._base
    end
    return false
  end
  setmetatable(newClass, classMetatable)
  return newClass
end

--! \class TCore_Clock
--! \brief a clock
TCore_Clock = class()

--! \brief get the current time
function TCore_Clock.GetTimeNow()
  if os.gettimeofday then
    return os.gettimeofday()
  else
    return os.time()
  end
end

--! \brief constructor
function TCore_Clock.init(this,T0)
  if T0 then
    this.t0 = T0
  else
    this.t0 = TCore_Clock.GetTimeNow()
  end
end

--! \brief get time string
function TCore_Clock.getTimeStamp(this,T0)
  local t0
  if T0 then
    t0 = T0
  else
    t0 = this.t0
  end
  return os.date('%c %Z',t0)
end


--! \brief io to console
--! 
--! pseudo class (no methods, just to keep documentation tidy)
TCore_IO = class()
-- 
--! \brief write to stdout
function TCore_IO_write(Str)
  if (Str) then
    io.write(Str)
  end
end

--! \brief write to stdout
function TCore_IO_writeln(Str)
  if (Str) then
    io.write(Str)
  end
  io.write("\n")
end


--! \brief trims a string
function string_trim(Str)
  return Str:match("^%s*(.-)%s*$")
end

--! \brief split a string
--! 
--! \param Str
--! \param Pattern
--! \returns table of string fragments
function string_split(Str, Pattern)
  local splitStr = {}
  local fpat = "(.-)" .. Pattern
  local last_end = 1
  local str, e, cap = string.find(Str,fpat, 1)
  while str do
    if str ~= 1 or cap ~= "" then
      table.insert(splitStr,cap)
    end
    last_end = e+1
    str, e, cap = string.find(Str,fpat, last_end)
  end
  if last_end <= #Str then
    cap = string.sub(Str,last_end)
    table.insert(splitStr, cap)
  end
  return splitStr
end


--! \class TCore_Commandline
--! \brief reads/parses commandline
TCore_Commandline = class()

--! \brief constructor
function TCore_Commandline.init(this)
  this.argv = arg
  this.parsed = {}
  this.params = {}
end

--! \brief get value
function TCore_Commandline.getRaw(this,Key,Default)
  local val = this.argv[Key]
  if not val then
    val = Default
  end
  return val
end

-------------------------------
--! \brief file buffer
--! 
--! an input file buffer
TStream_Read = class()

--! \brief get contents of file
--! 
--! \param Filename name of file to read (or nil == stdin)
function 	TStream_Read.getContents(this,Filename)
  -- get lines from file
  local filecontents
  if Filename then
    -- syphon lines to our table
    --TCore_Debug_show_var('Filename',Filename)
    filecontents={}
    for line in io.lines(Filename) do
      table.insert(filecontents,line)
    end
  else
    -- get stuff from stdin as a long string (with crlfs etc)
    filecontents=io.read('*a')
    --  make it a table of lines
    filecontents = TString_split(filecontents,'[\n]') -- note this only works for unix files.
    Filename = 'stdin'
  end

  if filecontents then
    this.filecontents = filecontents
    this.contentsLen = #filecontents
    this.currentLineNo = 1
  end

  return filecontents
end

--! \brief get lineno
function TStream_Read.getLineNo(this)
  return this.currentLineNo
end

--! \brief get a line
function TStream_Read.getLine(this)
  local line
  if this.currentLine then
    line = this.currentLine
    this.currentLine = nil
  else
    -- get line
    if this.currentLineNo<=this.contentsLen then
      line = this.filecontents[this.currentLineNo]
      this.currentLineNo = this.currentLineNo + 1
    else
      line = ''
    end
  end
  return line
end

--! \brief save line fragment
function TStream_Read.ungetLine(this,LineFrag)
  this.currentLine = LineFrag
end

--! \brief is it eof?
function TStream_Read.eof(this)
  if this.currentLine or this.currentLineNo<=this.contentsLen then
    return false
  end
  return true
end

--! \brief output stream
TStream_Write = class()

--! \brief constructor
function TStream_Write.init(this)
  this.tailLine = {}
end

--! \brief write immediately
function TStream_Write.write(this,Str)
  TCore_IO_write(Str)
end

--! \brief write immediately
function TStream_Write.writeln(this,Str)
  TCore_IO_writeln(Str)
end

--! \brief write immediately
function TStream_Write.writelnComment(this,Str)
  TCore_IO_write('// ZZ: ')
  TCore_IO_writeln(Str)
end

--! \brief write to tail
function TStream_Write.writelnTail(this,Line)
  if not Line then
    Line = ''
  end
  table.insert(this.tailLine,Line)
end

--! \brief outout tail lines
function TStream_Write.write_tailLines(this)
  for k,line in ipairs(this.tailLine) do
    TCore_IO_writeln(line)
  end
  TCore_IO_write('// Lua2DoX new eof')
end

--! \brief input filter
TLua2DoX_filter = class()

--! \brief allow us to do errormessages
function TLua2DoX_filter.warning(this,Line,LineNo,Legend)
  this.outStream:writelnTail(
  '//! \todo warning! ' .. Legend .. ' (@' .. LineNo .. ')"' .. Line .. '"'
  )
end

--! \brief trim comment off end of string
--!
--! If the string has a comment on the end, this trims it off.
--!
local function TString_removeCommentFromLine(Line)
  local pos_comment = string.find(Line,'%-%-')
  local tailComment
  if pos_comment then
    Line = string.sub(Line,1,pos_comment-1)
    tailComment = string.sub(Line,pos_comment)
  end
  return Line,tailComment
end

--! \brief get directive from magic
local function getMagicDirective(Line)
  local macro,tail
  local macroStr = '[\\@]'
  local pos_macro = string.find(Line,macroStr)
  if pos_macro then
    --! ....\\ macro...stuff
    --! ....\@ macro...stuff
    local line = string.sub(Line,pos_macro+1)
    local space = string.find(line,'%s+')
    if space then
      macro = string.sub(line,1,space-1)
      tail  = string_trim(string.sub(line,space+1))
    else
      macro = line
      tail  = ''
    end
  end
  return macro,tail
end

--! \brief check comment for fn
local function checkComment4fn(Fn_magic,MagicLines)
  local fn_magic = Fn_magic
  --	TCore_IO_writeln('// checkComment4fn "' .. MagicLines .. '"')

  local magicLines = string_split(MagicLines,'\n')

  local macro,tail

  for k,line in ipairs(magicLines) do
    macro,tail = getMagicDirective(line)
    if macro == 'fn' then
      fn_magic = tail
      --	TCore_IO_writeln('// found fn "' .. fn_magic .. '"')
    else
      --TCore_IO_writeln('// not found fn "' .. line .. '"')
    end
  end

  return fn_magic
end
--! \brief run the filter
function TLua2DoX_filter.readfile(this,AppStamp,Filename)
  local err

  local inStream = TStream_Read()
  local outStream = TStream_Write()
  this.outStream = outStream -- save to this obj

  if (inStream:getContents(Filename)) then
    -- output the file
    local line
    local fn_magic -- function name/def from  magic comment

    outStream:writelnTail('// #######################')
    outStream:writelnTail('// app run:' .. AppStamp)
    outStream:writelnTail('// #######################')
    outStream:writelnTail()

    local state = ''
    while not (err or inStream:eof()) do
      line = string_trim(inStream:getLine())
      -- 			TCore_Debug_show_var('inStream',inStream)
      -- 			TCore_Debug_show_var('line',line )
      if string.sub(line,1,2)=='--' then -- it's a comment
        if string.sub(line,3,3)=='@' then -- it's a magic comment
          state = 'in_magic_comment'
          local magic = string.sub(line,4)
          outStream:writeln('/// @' .. magic)
          fn_magic = checkComment4fn(fn_magic,magic)
        elseif string.sub(line,3,3)=='-' then -- it's a nonmagic doc comment
          local comment = string.sub(line,4)
          outStream:writeln('/// '.. comment)
        elseif string.sub(line,3,4)=='[[' then -- it's a long comment
          line = string.sub(line,5) -- nibble head
          local comment = ''
          local closeSquare,hitend,thisComment
          while (not err) and (not hitend) and (not inStream:eof()) do
            closeSquare = string.find(line,']]')
            if not closeSquare then -- need to look on another line
              thisComment = line .. '\n'
              line = inStream:getLine()
            else
              thisComment = string.sub(line,1,closeSquare-1)
              hitend = true

              -- unget the tail of the line
              -- in most cases it's empty. This may make us less efficient but
              -- easier to program
              inStream:ungetLine(string_trim(string.sub(line,closeSquare+2)))
            end
            comment = comment .. thisComment
          end
          if string.sub(comment,1,1)=='@' then -- it's a long magic comment
            outStream:write('/*' .. comment .. '*/  ')
            fn_magic = checkComment4fn(fn_magic,comment)
          else -- discard
            outStream:write('/* zz:' .. comment .. '*/  ')
            fn_magic = nil
          end
        -- TODO(justinmk): Uncomment this if we want "--" lines to continue the
        --                 preceding magic ("---", "--@", …) lines.
        -- elseif state == 'in_magic_comment' then  -- next line of magic comment
        --   outStream:writeln('/// '.. line:sub(3))
        else -- discard
          outStream:writeln('// zz:"' .. line .. '"')
          fn_magic = nil
        end
      elseif string.find(line,'^function') or string.find(line,'^local%s+function') then
        state = 'in_function'  -- it's a function
        local pos_fn = string.find(line,'function')
        -- function
        -- ....v...
        if pos_fn then
          -- we've got a function
          local fn_type
          if string.find(line,'^local%s+') then
            fn_type = ''--'static ' -- static functions seem to be excluded
          else
            fn_type = ''
          end
          local fn = TString_removeCommentFromLine(string_trim(string.sub(line,pos_fn+8)))
          if fn_magic then
            fn = fn_magic
          end

          if string.sub(fn,1,1)=='(' then
            -- it's an anonymous function
            outStream:writelnComment(line)
          else
            -- fn has a name, so is interesting

            -- want to fix for iffy declarations
            local open_paren = string.find(fn,'[%({]')
            if open_paren then
              -- we might have a missing close paren
              if not string.find(fn,'%)') then
                fn = fn .. ' ___MissingCloseParenHere___)'
              end
            end

            -- add vanilla function
            outStream:writeln(fn_type .. 'function ' .. fn .. '{}')
          end
        else
          this:warning(inStream:getLineNo(),'something weird here')
        end
        fn_magic = nil -- mustn't indavertently use it again
      else
        state = ''  -- unknown
        if #line>0 then  -- we don't know what this line means, so just comment it out
          outStream:writeln('// zz: ' .. line)
        else
          outStream:writeln() -- keep this line blank
        end
      end
    end

    -- output the tail
    outStream:write_tailLines()
  else
    outStream:writeln('!empty file')
  end
end

--! \brief this application
TApp = class()

--! \brief constructor
function TApp.init(this)
  local t0 = TCore_Clock()
  this.timestamp = t0:getTimeStamp()
  this.name = 'Lua2DoX'
  this.version = '0.2 20130128'
  this.copyright = 'Copyright (c) Simon Dales 2012-13'
end

function TApp.getRunStamp(this)
  return this.name .. ' (' .. this.version .. ') ' 
  .. this.timestamp
end

function TApp.getVersion(this)
  return this.name .. ' (' .. this.version .. ') ' 
end

function TApp.getCopyright(this)
  return this.copyright 
end

local This_app = TApp()

--main
local cl = TCore_Commandline()

local argv1 = cl:getRaw(2)
if argv1 == '--help' then
  TCore_IO_writeln(This_app:getVersion())
  TCore_IO_writeln(This_app:getCopyright())
  TCore_IO_writeln([[
  run as:
  lua2dox_filter <param>
  --------------
  Param:
  <filename> : interprets filename
  --version  : show version/copyright info
  --help     : this help text]])
elseif argv1 == '--version' then
  TCore_IO_writeln(This_app:getVersion())
  TCore_IO_writeln(This_app:getCopyright())
else
  -- it's a filter
  local appStamp = This_app:getRunStamp()
  local filename = argv1

  local filter = TLua2DoX_filter()
  filter:readfile(appStamp,filename)
end


--eof

lua2dox_filter

#!/bin/bash

###########################################################################
#   Copyright (C) 2012 by Simon Dales   #
#   simon@purrsoft.co.uk   #
#                                                                         #
#   This program is free software; you can redistribute it and/or modify  #
#   it under the terms of the GNU General Public License as published by  #
#   the Free Software Foundation; either version 2 of the License, or     #
#   (at your option) any later version.                                   #
#                                                                         #
#   This program is distributed in the hope that it will be useful,       #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of        #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         #
#   GNU General Public License for more details.                          #
#                                                                         #
#   You should have received a copy of the GNU General Public License     #
#   along with this program; if not, write to the                         #
#   Free Software Foundation, Inc.,                                       #
#   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             #
###########################################################################
LANG=""

##! \brief test executable to see if it exists
test_executable(){
	P_EXE="$1"
	#########
	WHICH=`which ${P_EXE}`
	if test -z "${WHICH}"
	then
		echo "not found \"${P_EXE}\""
	else
		EXE="${P_EXE}"
	fi
	}

##! \brief sets the lua interpreter
set_lua(){
	test_executable 'texlua'
	if test -z "${EXE}"
	then
		test_executable 'lua'
	fi
	#echo "final EXE=\"${EXE}\""
	}

##! \brief makes canonical name of file
##! 
##! Note that "readlink -f" doesn't work in MacOSX
##! 
do_readlink(){
	pushd . > /dev/null
	TARGET_FILE=$1
	
	cd `dirname $TARGET_FILE`
	TARGET_FILE=`basename $TARGET_FILE`
	
	# Iterate down a (possible) chain of symlinks
	while [ -L "$TARGET_FILE" ]
	do
		TARGET_FILE=`readlink $TARGET_FILE`
		cd `dirname $TARGET_FILE`
		TARGET_FILE=`basename $TARGET_FILE`
	done
	
	PHYS_DIR=`pwd -P`
	RESULT=$PHYS_DIR
	popd > /dev/null
	}

##main
set_lua
if test -z "${EXE}"
then
	echo "no lua interpreter available"
else
	BASENAME=`basename "$0"`
	do_readlink "$0"
	DIRNAME="${RESULT}"
		
	LUASCRIPT="${DIRNAME}/lua2dox.lua ${BASENAME}"
	#echo "lua[${LUASCRIPT}]"

	luajit ${LUASCRIPT} $@
fi
# 
#