Further formatting work.parser

7 files changed, 713 insertions, 121 deletions

diff --git a/muppet/format.py b/muppet/format.py
index b55d397..d72e5b2 100644
--- a/muppet/format.py
+++ b/muppet/format.py
@@ -11,9 +11,10 @@ import html
 import re
 from typing import (
     Tuple,
-    Any,
+    TypedDict,
+    cast,
+    Sequence,
 )
-import types
 
 from .puppet.parser import puppet_parser
 import logging
@@ -22,92 +23,508 @@ from .puppet.strings import (
     DataTypeAlias,
     DefinedType,
     DocString,
+    DocStringApiTag,
+    DocStringAuthorTag,
+    DocStringExampleTag,
+    DocStringOverloadTag,
+    DocStringOptionTag,
+    DocStringParamTag,
+    DocStringRaiseTag,
+    DocStringReturnTag,
+    DocStringSeeTag,
+    DocStringSinceTag,
+    DocStringSummaryTag,
+    DocStringTag,
     Function,
     PuppetClass,
     ResourceType,
-    DocStringParamTag,
-    DocStringExampleTag,
 )
+
 from muppet.puppet.ast import build_ast
 # from muppet.puppet.format import to_string
 from muppet.parser_combinator import (
     ParserCombinator,
     ParseError,
     MatchCompound,
+    MatchObject,
 )
 from muppet.puppet.format.parser import ParserFormatter
 
 
 logger = logging.getLogger(__name__)
 
-# parse_puppet = puppet_parser
-
 param_doc: dict[str, str] = {}
 
 
-def reserialize(obj: Any) -> str:
+def inner_text(obj: MatchObject | list[MatchObject]) -> str:
     """
-    Reconstruct puppet code after parsing it.
+    Extract the text content from a set of MatchObjects.
 
-    After building the parser, and parsing the puppet code into a tree
-    of MatchObjects; this procedure returns it into puppet code.
-    Difference being that we now have metadata, meaning that syntax
-    highlighting and variable hyperlinks can be inserted.
+    This is really similar to HTML's inner_text.
+
+    Empty whitespace tags are expanded into nothing, non-empty
+    whitespace tags becomes a single space (note that this discards
+    commets).
+
+    This only works properly if no function was mapped over the parser
+    return values in tree, see :func:`muppet.parser_combinator.fmap`.
 
     :param obj:
-        Should be assumed to be a list of MatchObject's, or something similar.
+        Match Objects to search.
+    """
+    match obj:
+        case str(s):
+            return s
+        case MatchCompound(type='ws', matched=[]):
+            return ''
+        case MatchCompound(type='ws'):
+            return ' '
+        case MatchCompound(matched=xs):
+            return ''.join(inner_text(x) for x in xs)
+        case [*xs]:
+            return ''.join(inner_text(x) for x in xs)
+        case _:
+            raise ValueError('How did we get here')
 
-        MatchCompound objects are serialized as
 
-        .. code-block:: html
+def name_to_url(name: str) -> tuple[str | None, str]:
+    """
+    Resolve a class or resource name into an url.
+
+    :param name:
+        The name of a class or resource, surch as "example::resource".
+    :return:
+        A tuple consisting of
+
+        - One of
+          - An internal link to the definition of that type
+          - A link to the official puppet documentation
+          - ``None``, if `name` is "class"
+        - A string indicating extra HTML classes for this url.
+          This is mostly so external references can be marked properly.
+    """
+    if name in built_in_types:
+        return (f'https://www.puppet.com/docs/puppet/7/types/{name}.html', 'puppet-doc')
+    elif name == 'class':
+        return (None, '')
+    else:
+        # TODO special cases for puppet's built in types.
+        # https://www.puppet.com/docs/puppet/7/cheatsheet_core_types.html
+        module, *items = name.lstrip(':').split('::')
+        # TODO get prefix from the command line/config file
+        return ('/code/muppet-strings/output/'
+                + '/'.join([module, 'manifests', *(items if items else ['init'])]),
+                '')
+
+
+puppet_doc_base = 'https://www.puppet.com/docs/puppet/7'
+lang_facts_builtin_variables = (f'{puppet_doc_base}/lang_facts_builtin_variables'
+                                '#lang_facts_builtin_variables')
+server_variables = f'{lang_facts_builtin_variables}-server-variables'
+compiler_variables = f'{lang_facts_builtin_variables}-compiler-variables'
+trusted_facts = f'{lang_facts_builtin_variables}-trusted-facts'
+server_facts = f'{lang_facts_builtin_variables}-server-facts'
+
+built_in_variables = {
+    'facts': 'https://google.com',
+    # clientcert, clientversion, puppetversion, clientnoop,
+    # agent_specified_environment:
+    # https://www.puppet.com/docs/puppet/7/lang_facts_builtin_variables#lang_facts_builtin_variables-agent-facts
+    'trusted': trusted_facts,
+    'server_facts': server_facts,
+    'environment': server_variables,
+    'servername': server_variables,
+    'serverip': server_variables,
+    'serverversion': server_variables,
+    'module_name': compiler_variables,
+    'caller_module_name': compiler_variables,
+
+    # Also note the special variable $title and $name
+    # https://www.puppet.com/docs/puppet/7/lang_defined_types#lang_defined_types-title-and-name
+}
+
+
+def parse_author(author: str) -> str:
+    """
+    Format author tags' content.
+
+    :param author:
+        The contents of the author tag. If the string is on the
+        regular "author" format of ``"Firstname Lastname
+        <first.last@example.com>"`` then the email will be formatted
+        and hyperlinked. Otherwise the string is returned verbatim.
+    :return:
+        An HTML safe string, possibly including tags.
+    """
+    m = re.match(r'(?P<author>.*) (<(?P<email>.*)>)|(?P<any>.*)', author)
+    assert m, "The above regex can't fail"
+    if m['author'] and m['email']:
+        author = html.escape(m['author'])
+        email = html.escape(m['email'])
+        return f'{author} <a class="email" href="mailto:{email}">&lt;{email}&gt</a>;'
+    else:
+        return html.escape(m['any'])
+
+
+# https://www.puppet.com/docs/puppet/7/cheatsheet_core_types.html
+# https://www.puppet.com/docs/puppet/7/types/file.html
+# ...
+built_in_types = {
+    'package',
+    'file',
+    'service',
+    'notify',
+    'exec',
+    'user',
+    'group',
+}
+
+# https://www.puppet.com/docs/puppet/7/function.html#{}
+built_in_functions = {
+    'abs',
+    'alert',
+    'all',
+    'annotate',
+    'any',
+    'assert_type',
+    'binary_file',
+    'break',
+    'call',
+    'camelcase',
+    'capitalize',
+    'ceiling',
+    'chomp',
+    'chop',
+    'compare',
+    'contain',
+    'convert_to',
+    'create_resources',
+    'crit',
+    'debug',
+    'defined',
+    'dig',
+    'digest',
+    'downcase',
+    'each',
+    'emerg',
+    'empty',
+    'epp',
+    'err',
+    'eyaml_lookup_key',
+    'fail',
+    'file',
+    'filter',
+    'find_file',
+    'find_template',
+    'flatten',
+    'floor',
+    'fqdn_rand',
+    'generate',
+    'get',
+    'getvar',
+    'group_by',
+    'hiera',
+    'hiera_array',
+    'hiera_hash',
+    'hiera_include',
+    'hocon_data',
+    'import',
+    'include',
+    'index',
+    'info',
+    'inline_epp',
+    'inline_template',
+    'join',
+    'json_data',
+    'keys',
+    'length',
+    'lest',
+    'lookup',
+    'lstrip',
+    'map',
+    'match',
+    'max',
+    'md5',
+    'min',
+    'module_directory',
+    'new',
+    'next',
+    'notice',
+    'partition',
+    'realize',
+    'reduce',
+    'regsubst',
+    'require',
+    'return',
+    'reverse_each',
+    'round',
+    'rstrip',
+    'scanf',
+    'sha1',
+    'sha256',
+    'shellquote',
+    'size',
+    'slice',
+    'sort',
+    'split',
+    'sprintf',
+    'step',
+    'strftime',
+    'strip',
+    'tag',
+    'tagged',
+    'template',
+    'then',
+    'tree_each',
+    'type',
+    'unique',
+    'unwrap',
+    'upcase',
+    'values',
+    'versioncmp',
+    'warning',
+    'with',
+    'yaml_data',
+}
+
+
+def find_declarations(objs: list[MatchObject]) -> list[str]:
+    """
+    Find all local variable declarations.
+
+    Searches the code for all local variable declarations, returing a
+    list of variable names.
 
-            <span class="{type}">{body}</span>
+    Note that the same variable might appear multiple times, for example:
 
-        strings as themselves, and lists have reserialize mapped over them.
+    .. code-block:: puppet
+        :caption: The same variable being declared twice
 
+        if $something {
+            $x = 10
+        } else {
+            $x = 20
+        }
     """
-    out: list[str] = []
-    # logger.info("obj = %a", obj)
-    match obj:
-        case str(s):
-            out += [s]
-        case MatchCompound(type=type, matched=xs):
-            # logger.warning("xs = %a", xs)
-            body = ''.join(reserialize(x) for x in xs)
-            out += [f'<span class="{type}">{body}</span>']
-        case [*xs]:
-            out += [reserialize(x for x in xs)]
-        case rest:
-            if isinstance(rest, types.GeneratorType):
-                out += [reserialize(x) for x in rest]
-            else:
-                logger.error("Unknown type: %a", rest)
+    declarations = []
+    for obj in objs:
+        match obj:
+            case MatchCompound(type='declaration', matched=xs):
+                for x in xs:
+                    match x:
+                        case MatchCompound(type='var', matched=ys):
+                            declarations.append(inner_text(ys))
+    return declarations
+
+
+class Reserializer:
+    """
+    Context for reserializing parsed data back into code.
 
-    return ''.join(out)
+    :param local_vars:
+        Variables declared within this file. Used when resolving
+        hyperlinks.
+    """
 
+    def __init__(self, local_vars: list[str]):
+        self.local_vars: list[str] = local_vars
+
+    def reserialize(self,
+                    obj: MatchObject | Sequence[MatchObject]) -> str:
+        """
+        Reconstruct puppet code after parsing it.
+
+        After building the parser, and parsing the puppet code into a tree
+        of MatchObjects; this procedure returns it into puppet code.
+        Difference being that we now have metadata, meaning that syntax
+        highlighting and variable hyperlinks can be inserted.
+
+        :param obj:
+            Should be assumed to be a list of MatchObject's, or something similar.
+
+            MatchCompound objects are serialized as
+
+            .. code-block:: html
+
+                <span class="{type}">{body}</span>
+
+            strings as themselves, and lists have reserialize mapped over them.
+
+        """
+        out: list[str] = []
+        # logger.info("obj = %a", obj)
+
+        # TODO hyperlink functions.
+        # The problem is that a function can either be implemented in
+        # Puppet, or in Ruby. And Ruby functions' names aren't bound
+        # by the directory layout.
+        match obj:
+            case str(s):
+                out.append(html.escape(s))
+
+            case MatchCompound(type='resource-name', matched=xs):
+                name = inner_text(xs)
+                url, cls = name_to_url(name)
+                if url:
+                    out.append(f'<a href="{url}" class="resource-name {cls}">{name}</a>')
+                else:
+                    # TODO this is class, but the class name should
+                    # also be hyperlinked
+                    out.append(f'<span class="resource-name {cls}">{name}</span>')
+
+            case MatchCompound(type='invoke', matched=xs):
+                function = None
+                for x in xs:
+                    match x:
+                        case MatchCompound(type='qn', matched=ys):
+                            if function is None:
+                                function = inner_text(ys)
+                                if function in built_in_functions:
+                                    # class="qn"
+                                    url = f"https://www.puppet.com/docs/puppet/7/function.html#{function}"  # noqa: E501
+                                    tag = f'<a href="{url}" class="puppet-doc">{self.reserialize(ys)}</a>'  # noqa: E501
+                                    out.append(tag)
+                                else:
+                                    # TODO function to url
+                                    out.append(f'<span class="qn">{self.reserialize(ys)}</span>')
+                            else:
+                                if function == 'include':
+                                    url, cls = name_to_url(inner_text(ys))
+                                    # class="qn"
+                                    tag = f'<a href="{url}" class="{cls}">{self.reserialize(ys)}</a>'  # noqa: E501
+                                    out.append(tag)
+                                else:
+                                    out.append(self.reserialize(ys))
+                        case _:
+                            out.append(self.reserialize(x))
+
+            case MatchCompound(type='declaration', matched=xs):
+                for x in xs:
+                    match x:
+                        case MatchCompound(type='var', matched=ys):
+                            inner = ''.join(self.reserialize(y) for y in ys)
+                            out.append(f'<span id="{inner_text(ys)}">{inner}</span>')
+                        case _:
+                            out.append(self.reserialize(x))
+
+            case MatchCompound(type='var', matched=xs):
+                out.append(self.var_to_url(inner_text(xs)))
+
+            case MatchCompound(type=type, matched=xs):
+                body = ''.join(self.reserialize(x) for x in xs)
+                out.append(f'<span class="{type}">{body}</span>')
+
+            case [*xs]:
+                out.extend(self.reserialize(x) for x in xs)
+
+            case rest:
+                logger.error("Unknown type: %a", rest)
 
-def parse_puppet(source: str, file: str) -> str:
+        return ''.join(out)
+
+    def var_to_url(self, var: str) -> str:
+        """
+        Format variable, adding hyperlink to its definition.
+
+        TODO these can refer to both defined types (`manifests/*.pp`),
+        as well as resource types (`lib/puppet/provider/*/*.rb` /
+        `lib/tpuppet/type/*.rb`)
+
+        Same goes for functions (`functions/*.pp`),
+        (`lib/puppet/functions.rb`).
+
+        :param var:
+            Name of the variable.
+
+        :return:
+            An HTML anchor element.
+        """
+        match var.split('::'):
+            case [name]:
+                # Either a local or global variable
+                # https://www.puppet.com/docs/puppet/7/lang_facts_and_builtin_vars.html
+
+                href = None
+                cls = ''
+                if name in self.local_vars:
+                    href = f'#{html.escape(var)}'
+                elif name in built_in_variables:
+                    href = html.escape(built_in_variables[name])
+                    cls = 'puppet-doc'
+
+                if href:
+                    return f'<a class="var {cls}" href="{href}">{var}</a>'
+                else:
+                    # `name` refers to a global fact.
+                    return f'<span class="var">{var}</span>'
+
+            case ['', name]:
+                # A global variable
+                if name in built_in_variables:
+                    href = html.escape(built_in_variables[name])
+                    img = '<img src="/code/muppet-strings/output/static/favicon.ico" />'
+                    return f'<a class="var" href="{href}">{var}{img}</a>'
+                else:
+                    return f'<span class="var">{var}</span>'
+
+            # Note the "special module" 'settings',
+            # https://www.puppet.com/docs/puppet/7/lang_facts_builtin_variables#lang_facts_builtin_variables-server-variables
+            case ['', module, *items, name]:
+                s = '/code/muppet-strings/output/' \
+                    + '/'.join([module, 'manifests', *(items if items else ['init'])])
+                s += f'#{name}'
+                return f'<a class="var" href="{s}">{var}</a>'
+            case [module, *items, name]:
+                s = '/code/muppet-strings/output/' \
+                    + '/'.join([module, 'manifests', *(items if items else ['init'])])
+                s += f'#{name}'
+                return f'<a class="var" href="{s}">{var}</a>'
+            case _:
+                raise ValueError()
+
+
+def parse_puppet(source: str, file: str, in_parameters: list[str]) -> str:
     """
     Parse and syntax highlight the given puppet source.
 
     :returns: An HTML string
     """
-    # logger.debug("source: %a", source)
     # Run the upstream puppet parser,
     # then masage the tree into a usable form.
     ast = build_ast(puppet_parser(source))
-    # logger.info("ast: %a", ast)
+
     # From the ast, build a parser combinator parser.
+    # This parser will attach sufficient metadata to allow syntax
+    # highlighting and hyperlinking
     parser = ParserFormatter().serialize(ast)
-    # logger.warning("parser: %a", parser)
-    # Run the generatefd parser, giving us a list of match objects
+
+    # Run the generated parser, giving us a list of match objects.
     match_objects = ParserCombinator(source, file).get(parser)
-    # logger.error("match_objects: %a", match_objects)
-    return reserialize(match_objects)
+
+    # Reserialize the matched data back into puppet code, realizing
+    # the syntax highlighting and hyperlinks.
+    return Reserializer(find_declarations(match_objects) + (in_parameters)) \
+        .reserialize(match_objects)
 
 # --------------------------------------------------
 
 
+GroupedTags = TypedDict('GroupedTags', {
+    'param': list[DocStringParamTag],
+    'example': list[DocStringExampleTag],
+    'overload': list[DocStringOverloadTag],
+    'option': dict[str, list[DocStringOptionTag]],
+    'author': list[DocStringAuthorTag],
+    'api': list[DocStringApiTag],
+    'raise': list[DocStringRaiseTag],
+    'return': list[DocStringReturnTag],
+    'since': list[DocStringSinceTag],
+    'summary': list[DocStringSummaryTag],
+    'see': list[DocStringSeeTag],
+    '_other': list[DocStringTag],
+})
+
+
 def format_docstring(name: str, docstring: DocString) -> Tuple[str, str]:
     """
     Format docstrings as they appear in some puppet types.
@@ -120,47 +537,165 @@ def format_docstring(name: str, docstring: DocString) -> Tuple[str, str]:
     """
     global param_doc
 
+    # The api tag is ignored, since it instead is shown from context
+
     out = ''
 
     param_doc = {tag.name: tag.text or ''
                  for tag in docstring.tags
                  if isinstance(tag, DocStringParamTag)}
-    tags = docstring.tags
 
-    # print(param_doc, file=sys.stderr)
+    grouped_tags: GroupedTags = {
+        'param': [],
+        'example': [],
+        'overload': [],
+        'option': {},
+        'author': [],
+        'api': [],
+        'raise': [],
+        'return': [],
+        'since': [],
+        'summary': [],
+        'see': [],
+        '_other': [],
+    }
+
+    for tag in docstring.tags:
+        if tag.tag_name == 'option':
+            tag = cast(DocStringOptionTag, tag)
+            grouped_tags['option'].setdefault(tag.parent, []).append(tag)
+        elif tag.tag_name in {'param', 'example', 'overload', 'author', 'api',
+                              'raise', 'return', 'since', 'summary', 'see'}:
+            grouped_tags[tag.tag_name].append(tag)  # type: ignore
+        else:
+            grouped_tags['_other'].append(tag)
 
-    # param_defaults = d_type['defaults']
+    # --------------------------------------------------
 
-    for t in tags:
-        text = html.escape(t.text)
-        if t.tag_name == 'summary':
-            out += '<em class="summary">'
-            out += text
-            out += '</em>'
+    out += '<a href="#code">Jump to Code</a><br/>'
 
-    for t in tags:
-        text = html.escape(t.text)
-        if isinstance(t, DocStringExampleTag):
-            if name := t.name:
-                out += f'<h3>{name}</h3>\n'
-            # TODO highlight?
-            out += f'<pre class="example"><code class="puppet">{text}</code></pre>\n'
-
-    # out += '<dl>'
-    # for t in tags:
-    #     if t['tag_name'] == 'param':
-    #         out += f"<dt>{t['name']}</dt>"
-    #         if text := t.get('text'):
-    #             text = re.sub(r'(NOTE|TODO)',
-    #                           r'<mark>\1</mark>',
-    #                           markdown(text))
-    #             out += f"<dd>{text}</dd>"
-    # out += '</dl>'
-
-    out += '<div>'
+    if tags := grouped_tags['summary']:
+        out += '<em class="summary">'
+        for tag in tags:
+            out += html.escape(tag.text)
+        out += '</em>'
+
+    out += '<div class="description">'
+    # TODO "TODO" highlighting
     out += markdown(docstring.text)
     out += '</div>'
 
+    # TODO proper handling of multiple @see tags
+    if sees := grouped_tags['see']:
+        out += '<b>See</b> '
+        for see in sees:
+            link: str
+            m = re.match(r'((?P<url>https?://.*)|(?P<man>.*\([0-9]\))|(?P<other>.*))', see.name)
+            assert m, "Regex always matched"
+            if m['url']:
+                link = f'<a href="{see.name}">{see.name}</a>'
+                out += link
+            elif m['man']:
+                page = see.name[:-3]
+                section = see.name[-2]
+                # TODO man providers
+                link = f"https://manned.org/man/{page}.{section}"
+                out += link
+            else:
+                if '::' in m['other']:
+                    # TODO
+                    pass
+                else:
+                    # TODO
+                    link = see
+                out += m['other']
+            out += ' ' + see.text
+
+    if authors := grouped_tags['author']:
+        out += '<div class="author">'
+        out += "<em>Written by </em>"
+        if len(authors) == 1:
+            out += parse_author(authors[0].text)
+        else:
+            out += '<ul>'
+            for author in authors:
+                out += f'<li>{parse_author(author.text)}</li>'
+            out += '</ul>'
+        out += '</div>'
+
+    out += '<hr/>'
+
+    t: DocStringTag
+
+    for t in grouped_tags['example']:
+        out += '<div class="code-example">'
+
+        if name := t.name:
+            # TODO markup for title
+            out += f'<div class="code-example-header">{html.escape(name)}</div>\n'
+        # TODO highlight?
+        # Problem is that we don't know what language the example
+        # is in. Pygemntize however seems to do a reasonable job
+        # treating anything as puppet code
+        text = html.escape(t.text)
+        out += f'<pre><code class="puppet">{text}</code></pre>\n'
+        out += '</div>'
+
+    out += '<hr/>'
+
+    out += '<dl>'
+    for t in grouped_tags['param']:
+        name = html.escape(t.name)
+        out += f'<dt><span id="{name}" class="variable">{name}</span>'
+        match t.types:
+            case [x]:
+                # TODO highlight type?
+                out += f': <code>{html.escape(x)}</code>'
+            case [_, *_]:
+                raise ValueError("How did you get multiple types onto a parameter?")
+
+        # TODO Fetch default values from puppet strings output
+        # Then in javascript query Hiera to get the true "default"
+        # values for a given machine (somewhere have a setting for
+        # selecting machine).
+        out += '</dt>'
+
+        if text := t.text:
+            text = re.sub(r'(NOTE|TODO)',
+                          r'<mark>\1</mark>',
+                          markdown(text))
+
+            if options := grouped_tags['option'].get(t.name):
+                text += '<dl>'
+                for option in options:
+                    text += '<dt>'
+                    text += html.escape(option.opt_name)
+                    match option.opt_types:
+                        case [x]:
+                            text += f' [<code>{html.escape(x)}</code>]'
+                        case [_, *_]:
+                            raise ValueError("How did you get multiple types onto an option?")
+                    text += '</dt>'
+                    text += '<dd>'
+                    if option.opt_text:
+                        text += re.sub(r'(NOTE|TODO)',
+                                       r'<mark>\1</mark>',
+                                       markdown(option.opt_text))
+                    text += '</dd>'
+                text += '</dl>'
+
+            out += f"<dd>{text}</dd>"
+        else:
+            out += '<dd><em>Undocumented</em></dd>'
+    out += '</dl>'
+
+    # TODO remaining tags
+    # "overload"
+    # raise
+    # return
+    # since
+    # _other
+
     return (name, out)
 
 
@@ -203,8 +738,16 @@ def format_class(d_type: DefinedType | PuppetClass) -> Tuple[str, str]:
     # renderer = HTMLRenderer(build_param_dict(d_type.docstring))
     # out += render(renderer, data)
     # ------ New ---------------------------------------
+    out += '<hr/>'
+    out += '<a id="code"></a>'
+
+    in_parameters: list[str] = []
+    for tag in d_type.docstring.tags:
+        if tag.tag_name == 'param':
+            in_parameters.append(cast(DocStringParamTag, tag).name)
+
     try:
-        result = parse_puppet(d_type.source, d_type.name)
+        result = parse_puppet(d_type.source, d_type.name, in_parameters)
         out += '<pre class="highlight-muppet"><code class="puppet">'
         out += result
         out += '</code></pre>'
@@ -241,7 +784,7 @@ def format_type_alias(d_type: DataTypeAlias, file: str) -> Tuple[str, str]:
     out += '\n'
     out += '<pre class="highlight-muppet"><code class="puppet">'
     try:
-        out += parse_puppet(d_type.alias_of, file)
+        out += parse_puppet(d_type.alias_of, file, [])
     except ParseError as e:
         logger.error("Parsing %(name)s failed: %(err)s",
                      {'name': d_type.alias_of, 'err': e})
@@ -261,7 +804,7 @@ def format_defined_type(d_type: DefinedType, file: str) -> Tuple[str, str]:
 
     out += '<pre class="highlight-muppet"><code class="puppet">'
     try:
-        out += parse_puppet(d_type.source, file)
+        out += parse_puppet(d_type.source, file, [])
     except ParseError as e:
         logger.error("Parsing %(name)s failed: %(err)s",
                      {'name': d_type.source, 'err': e})
@@ -326,7 +869,7 @@ def format_puppet_function(function: Function, file: str) -> str:
     elif t == 'puppet':
         out += '<pre class="highlight-muppet"><code class="puppet">'
         try:
-            out += parse_puppet(function.source, file)
+            out += parse_puppet(function.source, file, [])
         except ParseError as e:
             logger.error("Parsing %(name)s failed: %(err)s",
                          {'name': function.source, 'err': e})
diff --git a/muppet/output.py b/muppet/output.py
index 08239c9..de285cb 100644
--- a/muppet/output.py
+++ b/muppet/output.py
@@ -69,6 +69,7 @@ class ResourceTypeOutput:
     """Basic HTML implementation."""
 
     title: str
+    module_name: str
     children: list['HtmlSerializable'] = field(default_factory=list)
     link: Optional[str] = None
     summary: Optional[str] = None
@@ -90,7 +91,9 @@ class ResourceTypeOutput:
         # self.__class__.__name__
         return jinja \
             .get_template('snippets/ResourceType-index-entry.html') \
-            .render(item=self)
+            .render(item=self,
+                    module_name=self.module_name,
+                    prefix='/code/muppet-strings/output')
 
     def to_html_list(self) -> str:
         """Return HTML suitable for a list."""
@@ -361,7 +364,8 @@ class ResourceIndex:
         return out
 
 
-def resource_type_index(resource_types: list[ResourceType]) -> list[HtmlSerializable]:
+def resource_type_index(resource_types: list[ResourceType],
+                        module_name: str) -> list[HtmlSerializable]:
     """Generate index for all known resource types."""
     lst: list[HtmlSerializable] = []
 
@@ -378,9 +382,11 @@ def resource_type_index(resource_types: list[ResourceType]) -> list[HtmlSerializ
                 items.append(ResourceTypeOutput(
                     title=provider.name,
                     link=provider.file,
+                    module_name=module_name,
                     summary=documentation))
 
         lst.append(ResourceTypeOutput(title=resource_type.name,
+                                      module_name=module_name,
                                       children=items))
 
     return lst
@@ -420,7 +426,7 @@ def setup_module_index(*,
 
     content.append(ResourceIndex(
         title='Resource Types',
-        children=resource_type_index(data.resource_types)))
+        children=resource_type_index(data.resource_types, module.name)))
 
     # data['providers']
     content.append(IndexCategory(
@@ -549,7 +555,7 @@ def setup_module(base: str, module: ModuleEntry, *, path_base: str) -> None:
         title, body = format_class(puppet_class)
         with open(os.path.join(dir, 'index.html'), 'w') as f:
             f.write(templates.code_page(
-                title=title,
+                title=module.name,
                 content=body,
                 path_base=path_base,
                 breadcrumbs=crumbs))
diff --git a/muppet/puppet/format/parser.py b/muppet/puppet/format/parser.py
index aaffae3..8366612 100644
--- a/muppet/puppet/format/parser.py
+++ b/muppet/puppet/format/parser.py
@@ -210,7 +210,7 @@ class ParserFormatter(Serializer[ParseDirective]):
 
         type = self.s(item.type)
         value = optional(ws & '=' & ws & self.s(item.v))
-        return name(f'decl-${item.k}', ws & type & ws & '$' & item.k & value)
+        return name(f'decl-${item.k}', ws & type & ws & '$' & tag('declaration', item.k) & value)
 
     def instanciation_parameter(self, param: PuppetInstanciationParameter) -> ParseDirective:
         """
@@ -331,15 +331,15 @@ class ParserFormatter(Serializer[ParseDirective]):
 
     @override
     def _puppet_access(self, it: PuppetAccess) -> ParseDirective:
-        return tag('access', ws & self.s(it.how) & ws & self.known_array('[]', it.args))
+        return ws & tag('access', self.s(it.how) & ws & self.known_array('[]', it.args))
 
     @override
     def _puppet_array(self, it: PuppetArray) -> ParseDirective:
-        return tag('array', ws & self.known_array('[]', it.items))
+        return ws & tag('array', self.known_array('[]', it.items))
 
     @override
     def _puppet_binary_operator(self, it: PuppetBinaryOperator) -> ParseDirective:
-        return ws & self.s(it.lhs) & ws & it.op & ws & self.s(it.rhs)
+        return ws & self.s(it.lhs) & ws & tag('op', it.op) & ws & self.s(it.rhs)
 
     @override
     def _puppet_block(self, it: PuppetBlock) -> ParseDirective:
@@ -347,15 +347,17 @@ class ParserFormatter(Serializer[ParseDirective]):
 
     @override
     def _puppet_call(self, it: PuppetCall) -> ParseDirective:
-        return ws & self.s(it.func) & \
-                optional(ws & self.known_array('()', it.args)) & \
-                optional(ws & self.s(it.block))
+        return ws & tag('call',
+                        self.s(it.func) &
+                        optional(ws & self.known_array('()', it.args)) &
+                        optional(ws & self.s(it.block)))
 
     @override
     def _puppet_call_method(self, it: PuppetCallMethod) -> ParseDirective:
-        return ws & self.s(it.func) & \
-                optional(ws & self.known_array('()', it.args)) & \
-                optional(ws & self.s(it.block))
+        return ws & tag('call-method',
+                        self.s(it.func) &
+                        optional(ws & self.known_array('()', it.args)) &
+                        optional(ws & self.s(it.block)))
 
     @override
     def _puppet_case(self, it: PuppetCase) -> ParseDirective:
@@ -373,7 +375,7 @@ class ParserFormatter(Serializer[ParseDirective]):
     def _puppet_class(self, it: PuppetClass) -> ParseDirective:
         parser = (ws & tag('keyword', 'class') & ws & tag('name', it.name) &
                   optional(ws & self.declaration_parameters('()', it.params)))
-        parser &= optional(ws & 'inherits' & ws & tag('inherits', it.parent))
+        parser &= optional(ws & tag('keyword', 'inherits') & ws & tag('inherits', it.parent))
         parser &= ws & '{' & ws & self.s(it.body) & ws & '}'
         return parser
 
@@ -391,7 +393,7 @@ class ParserFormatter(Serializer[ParseDirective]):
 
     @override
     def _puppet_concat(self, it: PuppetConcat) -> ParseDirective:
-        parser = ws & '"'
+        parser: ParseDirective = s('"')
         for fragment in it.fragments:
             match fragment:
                 case PuppetString(st):
@@ -400,15 +402,15 @@ class ParserFormatter(Serializer[ParseDirective]):
                 case _:
                     parser &= interpolated_form(self.s(fragment))
         parser &= s('"') & ws
-        return parser
+        return ws & tag('string', parser)
 
     @override
     def _puppet_declaration(self, it: PuppetDeclaration) -> ParseDirective:
-        # TODO tag with declaration
-        return ws & self.s(it.k) & ws & '=' & ws & self.s(it.v)
+        return ws & tag('declaration', self.s(it.k)) & ws & '=' & ws & self.s(it.v)
 
     @override
     def _puppet_define(self, it: PuppetDefine) -> ParseDirective:
+        # TODO tag name with something
         return (ws & tag('keyword', 'define') & ws & it.name &
                 optional(ws & self.declaration_parameters('()', it.params)) &
                 ws & '{' & ws & self.s(it.body) & ws & '}')
@@ -451,14 +453,14 @@ class ParserFormatter(Serializer[ParseDirective]):
         This will however not give any false positives, since our
         parser is built from the source.
         """
-        parser = ws & optional(s('{'))
+        parser = optional(s('{'))
         for entry in it.entries:
             parser &= (ws & self.s(entry.k) &
-                       ws & '=>' &
+                       ws & tag('op', '=>') &
                        ws & self.s(entry.v) &
                        optional(ws & ','))
         parser &= ws & optional(s('}'))
-        return parser
+        return ws & tag('hash', parser)
 
     @override
     def _puppet_if_chain(self, it: PuppetIfChain) -> ParseDirective:
@@ -497,7 +499,7 @@ class ParserFormatter(Serializer[ParseDirective]):
 
     @override
     def _puppet_keyword(self, it: PuppetKeyword) -> ParseDirective:
-        return tag('keyword', ws & it.name)
+        return ws & tag('keyword', it.name)
 
     @override
     def _puppet_lambda(self, it: PuppetLambda) -> ParseDirective:
@@ -507,7 +509,7 @@ class ParserFormatter(Serializer[ParseDirective]):
 
     @override
     def _puppet_literal(self, it: PuppetLiteral) -> ParseDirective:
-        return tag('literal', ws & it.literal)
+        return ws & tag('literal', it.literal)
 
     @override
     def _puppet_heredoc(self, it: PuppetHeredoc) -> ParseDirective:
@@ -561,7 +563,7 @@ class ParserFormatter(Serializer[ParseDirective]):
 
     @override
     def _puppet_node(self, it: PuppetNode) -> ParseDirective:
-        parser = ws & 'node' & ws
+        parser = ws & tag('keyword', 'node') & ws
         for match in it.matches:
             parser &= ws & match & ws & ","
         parser &= ws & "{" & ws & self.s(it.body) & "}"
@@ -573,18 +575,18 @@ class ParserFormatter(Serializer[ParseDirective]):
 
     @override
     def _puppet_number(self, it: PuppetNumber) -> ParseDirective:
-        parser: ParseDirective = ws
+        parser: ParseDirective
         match (it.x, it.radix):
             case int(x), 8:
-                parser &= s('0') & oct(x)[2:]
+                parser = s('0') & oct(x)[2:]
             case int(x), 16:
-                parser &= s('0') & 'x' & hex(x)[2:]
+                parser = s('0') & 'x' & hex(x)[2:]
             case x, None:
-                parser &= str(x)
+                parser = s(str(x))
             case _:
                 raise ValueError(f"Unexpected radix: {it.radix}")
 
-        return parser
+        return ws & tag('number', parser)
 
     @override
     def _puppet_parenthesis(self, it: PuppetParenthesis) -> ParseDirective:
@@ -592,15 +594,15 @@ class ParserFormatter(Serializer[ParseDirective]):
 
     @override
     def _puppet_qn(self, it: PuppetQn) -> ParseDirective:
-        return tag('qn', ws & it.name)
+        return ws & tag('qn', it.name)
 
     @override
     def _puppet_qr(self, it: PuppetQr) -> ParseDirective:
-        return tag('qr', ws & it.name)
+        return ws & tag('qr', it.name)
 
     @override
     def _puppet_regex(self, it: PuppetRegex) -> ParseDirective:
-        return tag('rx', ws & '/' & it.s.replace('/', r'\/') & '/')
+        return ws & tag('rx', s('/') & it.s.replace('/', r'\/') & '/')
 
     @override
     def _puppet_resource(self, it: PuppetResource) -> ParseDirective:
@@ -611,17 +613,19 @@ class ParserFormatter(Serializer[ParseDirective]):
             case 'exported':
                 parser &= '@@'
 
-        parser &= ws & self.s(it.type) & ws & '{'
+        parser &= ws & tag('resource-name', self.s(it.type)) & ws & '{'
+        # TODO tag things here
         for key, params in it.bodies:
             parser &= ws & self.s(key) & ws & ':'
             for param in params:
                 parser &= self.instanciation_parameter(param)
             parser &= ws & optional(s(';'))
         parser &= ws & '}'
-        return parser
+        return tag('resource',  parser)
 
     @override
     def _puppet_resource_defaults(self, it: PuppetResourceDefaults) -> ParseDirective:
+        # TODO tag things here
         parser = ws & self.s(it.type) & ws & '{' & ws
         for param in it.ops:
             parser &= self.instanciation_parameter(param)
@@ -630,6 +634,7 @@ class ParserFormatter(Serializer[ParseDirective]):
 
     @override
     def _puppet_resource_override(self, it: PuppetResourceOverride) -> ParseDirective:
+        # TODO tag things here
         parser = ws & self.s(it.resource) & ws & '{' & ws
         for param in it.ops:
             parser &= self.instanciation_parameter(param)
@@ -638,6 +643,7 @@ class ParserFormatter(Serializer[ParseDirective]):
 
     @override
     def _puppet_selector(self, it: PuppetSelector) -> ParseDirective:
+        # TODO tag things here
         parser = ws & self.s(it.resource) & ws & '?' & ws & '{'
         for key, body in it.cases:
             parser &= ws & self.s(key) & ws & '=>' & ws & self.s(body) & ws & optional(s(','))
@@ -669,21 +675,20 @@ class ParserFormatter(Serializer[ParseDirective]):
 
     @override
     def _puppet_unary_operator(self, it: PuppetUnaryOperator) -> ParseDirective:
-        return ws & it.op & ws & self.s(it.x)
+        return ws & tag('op', it.op) & ws & self.s(it.x)
 
     @override
     def _puppet_unless(self, it: PuppetUnless) -> ParseDirective:
-        parser = (ws & 'unless' & ws & self.s(it.condition) & ws & '{' &
+        parser = (ws & tag('keyword', 'unless') & ws & self.s(it.condition) & ws & '{' &
                   ws & self.s(it.consequent) & ws & '}')
-        parser &= optional(ws & 'else' & ws & '{' & ws & self.s(it.alternative) &
+        parser &= optional(ws & tag('keyword', 'else') &
+                           ws & '{' &
+                           ws & self.s(it.alternative) &
                            ws & '}')
         return parser
 
     @override
     def _puppet_var(self, it: PuppetVar) -> ParseDirective:
-        # TODO highlight entire decalaration
-        # TODO hyperlink?
-
         # The leading '$' is optional, since it's optional for
         # variables in string interpolations, e.g. "${x}".
         return name(f'${it.name}', ws & optional(s('$')) & tag('var', it.name))
diff --git a/static-src/style.scss b/static-src/style.scss
index ec677d7..70d6ee9 100644
--- a/static-src/style.scss
+++ b/static-src/style.scss
@@ -77,10 +77,29 @@ code.json {
 	display: inline;
 }
 
-.example {
-	background: lightgray;
-	padding: 1em;
+.code-example {
+	/* Lighter Light gray */
+	background: #edecea;
 	border-radius: $border-radius;
+	overflow: hidden;
+	margin: 1em;
+
+	pre {
+		padding: 1em;
+		padding-top: 1pt;
+		padding-bottom: 1pt;
+	}
+
+
+	.code-example-header {
+		background: beige;
+		/* Dark beige */
+		border-bottom: 2px solid #b2b2a0;
+		padding: 1em;
+		padding-bottom: 0;
+		padding-top: 0;
+		font-family: sans;
+	}
 }
 
 .comment {
@@ -196,6 +215,20 @@ span.error {
 	color: white;
 }
 
+code .puppet-doc {
+	color: orange;
+}
+
+.email {
+	font-family: mono;
+	font-size: 80%;
+}
+
+
+dt .variable {
+	font-weight: bold;
+}
+
 /* -------------------------------------------------- */
 
 @import "colorscheme_default";
diff --git a/templates/base.html b/templates/base.html
index 9c8fa46..d116b16 100644
--- a/templates/base.html
+++ b/templates/base.html
@@ -13,7 +13,7 @@ Parameters:
 		An optional list of breadcrumb items.
 #}
 <!doctype html>
-<html>
+<html lang="en">
 	<head>
 		<meta charset="UTF-8">
 		<meta name="viewport" content="width=device-width, initial-scale=1.0">
@@ -30,6 +30,7 @@ Parameters:
 }
 			</style>
 		</noscript>
+		<title>{% block title %}Muppet{% endblock %}</title>
 	</head>
 	<body>
 		<header>
diff --git a/templates/code_page.html b/templates/code_page.html
index 2e7f99c..5b2d364 100644
--- a/templates/code_page.html
+++ b/templates/code_page.html
@@ -19,7 +19,6 @@ Parameters:
 		<li><a href="index.html">Rendered</a></li>
 		<li><a href="source.pp.html">Source</a></li>
 		<li><a href="source.pp.txt">Raw Source</a></li>
-		<li><a href="source.json">JSON blob</a></li>
 	</ul>
 {{ content }}
 {% endblock %}
diff --git a/templates/snippets/ResourceType-index-entry.html b/templates/snippets/ResourceType-index-entry.html
index b08d658..45501f4 100644
--- a/templates/snippets/ResourceType-index-entry.html
+++ b/templates/snippets/ResourceType-index-entry.html
@@ -1,6 +1,11 @@
 {#
+:param item: An instance of ResourceTypeOutput
+:param prefix: Prefix for HTTP output path,
+	(e.g. '/code/muppet-strings/output')
+:param module_name:
+
 #}
-<dt><a href="#">{{ item.base() }}</a></dt>
+<dt><a href="{{ prefix }}/{{ module_name }}/lib/puppet/types/{{ item.base() }}.rb">{{ item.base() }}</a></dt>
 <dd>
 	{% if item.summary %}
 		<!-- TODO docstring.text -->