diff options
| -rw-r--r-- | muppet/data/__init__.py | 202 | ||||
| -rw-r--r-- | muppet/data/html.py | 95 | ||||
| -rw-r--r-- | muppet/data/plain.py | 50 | ||||
| -rw-r--r-- | muppet/format.py | 1037 | ||||
| -rw-r--r-- | muppet/puppet/__main__.py | 72 | ||||
| -rw-r--r-- | muppet/puppet/ast.py | 724 | ||||
| -rw-r--r-- | muppet/puppet/format/__init__.py | 13 | ||||
| -rw-r--r-- | muppet/puppet/format/base.py | 296 | ||||
| -rw-r--r-- | muppet/puppet/format/html.py | 549 | ||||
| -rw-r--r-- | muppet/puppet/format/text.py | 594 | ||||
| -rw-r--r-- | muppet/puppet/parser.py | 33 | ||||
| -rw-r--r-- | tests/test_ast.py | 633 | ||||
| -rw-r--r-- | tests/test_parse.py | 538 |
13 files changed, 2916 insertions, 1920 deletions
diff --git a/muppet/data/__init__.py b/muppet/data/__init__.py deleted file mode 100644 index 6666e5b..0000000 --- a/muppet/data/__init__.py +++ /dev/null @@ -1,202 +0,0 @@ -""" -Data types representing a tagged document. - -Also contains the function signatures for rendering the document into a concrete -representation (HTML, plain text, ...) - -Almost all of the datatypes have "bad" __repr__ implementations. This -is to allow *much* easier ocular diffs when running pytest. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from abc import ABC, abstractmethod -from collections.abc import Sequence -from typing import ( - Any, - TypeAlias, - Union, -) - - -Markup: TypeAlias = Union[str, - 'Tag', - 'Declaration', - 'Link', - 'ID', - 'Documentation', - 'Indentation'] -""" -Documentation of Markup. -""" - - -@dataclass -class Tag: - """An item with basic metadata.""" - - # item: Any # str | 'Tag' | Sequence[str | 'Tag'] - item: Markup | Sequence[Markup] - # item: Any - tags: Sequence[str] - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, Tag): - return False - return self.item == other.item and set(self.tags) == set(other.tags) - - def __repr__(self) -> str: - return f'tag({repr(self.item)}, tags={self.tags})' - - -@dataclass -class Declaration(Tag): - """ - Superset of tag, containing declaration statements. - - Mostly used for class and resource parameters. - - :param variable: - Name of the variable being declared. - """ - - variable: str - - -@dataclass -class Link: - """An item which should link somewhere.""" - - item: Markup - target: str - - def __repr__(self) -> str: - return f'link({repr(self.item)})' - - -@dataclass -class ID: - """Item with an ID attached.""" - - item: Markup - id: str - - def __repr__(self) -> str: - return f'id({repr(self.item)})' - - -@dataclass -class Documentation: - """Attach documentation to a given item.""" - - item: Markup - documentation: str - - def __repr__(self) -> str: - return f'doc({repr(self.item)})' - - -@dataclass -class Indentation: - """Abstract indentation object.""" - - depth: int - - def __str__(self) -> str: - return ' ' * self.depth * 2 - - def __repr__(self) -> str: - if self.depth == 0: - return '_' - else: - return 'i' * self.depth - - -def tag(item: Markup | Sequence[Markup], *tags: str) -> Tag: - """Tag item with tags.""" - return Tag(item, tags=tags) - - -def declaration(item: Markup | Sequence[Markup], *tags: str, variable: str) -> Declaration: - """Mark name of variable in declaration.""" - return Declaration(item, tags=tags, variable=variable) - - -def link(item: Markup, target: str) -> Link: - """Create a new link element.""" - return Link(item, target) - - -def id(item: Markup, id: str) -> ID: - """Attach an id to an item.""" - return ID(item, id) - - -def doc(item: Markup, documentation: str) -> Documentation: - """Attach documentation to an item.""" - return Documentation(item, documentation) - - -class Renderer(ABC): - """ - Group of functions to render a marked up document into an output format. - - `render` could be a class method, but the base class wouldn't have any - members. Instead, have that as a standalone function, and give a renderer - "namespace" to it. - """ - - @abstractmethod - def render_tag(self, tag: Tag) -> str: - """Render a tag value into a string.""" - raise NotImplementedError - - @abstractmethod - def render_link(self, link: Link) -> str: - """Render a link value into a string.""" - raise NotImplementedError - - @abstractmethod - def render_id(self, id: ID) -> str: - """Render an object with a wrapping id tag into a string.""" - raise NotImplementedError - - @abstractmethod - def render_doc(self, doc: Documentation) -> str: - """Render an item with attached documentation into a string.""" - raise NotImplementedError - - @abstractmethod - def render_indent(self, i: Indentation) -> str: - """Render whitespace making out indentation.""" - raise NotImplementedError - - @abstractmethod - def render_str(self, s: str) -> str: - """ - Render plain strings into plain strings. - - This is needed since some implementations might want to do - something with whitespace, prettify operators, or anything - else. - """ - raise NotImplementedError - - -def render(renderer: Renderer, doc: Markup) -> str: - """Render a given document, with the help of a given renderer.""" - if isinstance(doc, Tag): - return renderer.render_tag(doc) - elif isinstance(doc, Link): - return renderer.render_link(doc) - elif isinstance(doc, ID): - return renderer.render_id(doc) - elif isinstance(doc, Documentation): - return renderer.render_doc(doc) - elif isinstance(doc, Indentation): - return renderer.render_indent(doc) - elif isinstance(doc, str): - return renderer.render_str(doc) - else: - raise ValueError('Value in tree of unknown value', doc) diff --git a/muppet/data/html.py b/muppet/data/html.py deleted file mode 100644 index df4f0f2..0000000 --- a/muppet/data/html.py +++ /dev/null @@ -1,95 +0,0 @@ -"""HTML Renderer.""" - -from . import ( - Tag, - Declaration, - Link, - ID, - Documentation, - Renderer, - Indentation, - render, -) -from collections.abc import Sequence -import html -from dataclasses import dataclass, field - - -@dataclass -class HTMLRenderer(Renderer): - """ - Render the document into HTML. - - :param param_documentation: - A dictionary containing (rendered) documentation for each - parameter of the class or resource type currently being - rendered. - """ - - param_documentation: dict[str, str] = field(default_factory=dict) - - def render_tag(self, tag: Tag) -> str: - """Attaches all tags as classes in a span.""" - inner: str - # if isinstance(tag.item, str): - # inner = tag.item - if isinstance(tag.item, Sequence): - inner = ''.join(render(self, i) for i in tag.item) - else: - inner = render(self, tag.item) - - out = '' - if isinstance(tag, Declaration): - if comment := self.param_documentation.get(tag.variable): - if isinstance(tag.item, list) \ - and tag.item \ - and isinstance(tag.item[0], Indentation): - out += render(self, tag.item[0]) - out += f'<span class="comment">{comment.strip()}</span>\n' - - tags = ' '.join(tag.tags) - out += f'<span class="{tags}">{inner}</span>' - return out - - def render_link(self, link: Link) -> str: - """Wrap the value in an anchor tag.""" - return f'<a href="{link.target}">{render(self, link.item)}</a>' - - def render_id(self, id: ID) -> str: - """Render an object with a wrapping id tag into a string.""" - return f'<span id="{id.id}">{render(self, id.item)}</span>' - - def render_doc(self, doc: Documentation) -> str: - """ - Set up a documentation anchor span, with content. - - The anchor will contain both the item, rendered as normally, - and a div with class documentation. - - The suggested CSS for this is:: - - .documentation-anchor { - display: relative; - } - - .documentation-anchor .documentation { - display: none; - } - - .documentation-anchor:hover .documentation { - display: block; - } - """ - s = '<span class="documentation-anchor">' - s += render(self, doc.item) - s += f'<div class="documentation">{doc.documentation}</div>' - s += '</span>' - return s - - def render_indent(self, ind: Indentation) -> str: - """Return indentation width * 2 as a string.""" - return ' ' * ind.depth * 2 - - def render_str(self, s: str) -> str: - """HTML escape and return the given string.""" - return html.escape(s) diff --git a/muppet/data/plain.py b/muppet/data/plain.py deleted file mode 100644 index 0ffdcf5..0000000 --- a/muppet/data/plain.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Plain text renderer.""" - -from . import ( - Tag, - Link, - ID, - Documentation, - Renderer, - Indentation, - render, -) -from collections.abc import Sequence - - -class TextRenderer(Renderer): - """ - Renders the document back into plain text. - - On its own this is rather worthless, since we already started with - valid Puppet code (altough this being prettified, but without - comments). It however allows us to feed our output back into puppet lint, - which will work nicely as end-to-end tests. - """ - - def render_tag(self, tag: Tag) -> str: - """Only renders the content.""" - if isinstance(tag.item, Sequence): - return ''.join(render(self, i) for i in tag.item) - else: - return render(self, tag.item) - - def render_link(self, link: Link) -> str: - """Only renders the content.""" - return render(self, link.item) - - def render_id(self, id: ID) -> str: - """Only renders the content.""" - return render(self, id.item) - - def render_doc(self, doc: Documentation) -> str: - """Only renders the content.""" - return render(self, doc.item) - - def render_indent(self, ind: Indentation) -> str: - """Return indentation width * 2 as a string.""" - return ' ' * ind.depth * 2 - - def render_str(self, s: str) -> str: - """Return the string verbatim.""" - return s diff --git a/muppet/format.py b/muppet/format.py index 734f8bd..26c0b8f 100644 --- a/muppet/format.py +++ b/muppet/format.py @@ -12,31 +12,11 @@ import html import sys import re from typing import ( - Any, - Literal, Tuple, - TypeAlias, - Union, ) from .puppet.parser import puppet_parser -from .intersperse import intersperse -from .data import ( - Markup, - Indentation, - Tag, - Link, - doc, - id, - link, - tag, - declaration, - render, -) - -from .data.html import ( - HTMLRenderer, -) +import logging from .puppet.strings import ( DataTypeAlias, @@ -48,987 +28,18 @@ from .puppet.strings import ( DocStringParamTag, DocStringExampleTag, ) +from muppet.puppet.ast import build_ast +from muppet.puppet.format import serialize +from muppet.puppet.format.html import HTMLFormatter -parse_puppet = puppet_parser -HashEntry: TypeAlias = Union[Tuple[Literal['=>'], str, Any], - Tuple[Literal['+>'], str, Any], - Tuple[Literal['splat-hash'], Any]] +logger = logging.getLogger(__name__) -Context: TypeAlias = list['str'] +# parse_puppet = puppet_parser param_doc: dict[str, str] = {} - -def ind(level: int) -> Indentation: - """Return a string for indentation.""" - return Indentation(level) - - -def keyword(x: str) -> Tag: - """Return a keyword token for the given string.""" - return tag(x, 'keyword', x) - - -def operator(op: str) -> Tag: - """Tag string as an operator.""" - return tag(op, 'operator') - - -def print_hash(hash: list[HashEntry], - indent: int, - context: Context) -> Tag: - """Print the contents of a puppet hash literal.""" - if not hash: - return tag('') - # namelen = 0 - items: list[Markup] = [] - for item in hash: - match item: - case ['=>', key, value]: - items += [ - ind(indent), - parse(key, indent, context), - ' ', '=>', ' ', - parse(value, indent, context), - ] - case _: - items += [tag(f'[|[{item}]|]', 'parse-error'), '\n'] - items += [',', '\n'] - - return tag(items) - - -def ops_namelen(ops: list[HashEntry]) -> int: - """Calculate max key length a list of puppet operators.""" - namelen = 0 - for item in ops: - match item: - case ['=>', key, _]: - namelen = max(namelen, len(key)) - case ['+>', key, _]: - namelen = max(namelen, len(key)) - case ['splat-hash', _]: - namelen = max(namelen, 1) - case _: - raise Exception("Unexpected item in resource:", item) - return namelen - - -def print_var(x: str, dollar: bool = True) -> Link: - """ - Print the given variable. - - If documentation exists, then add that documentation as hoover text. - - :param x: - The variable to print - :param dollar: - If there should be a dollar prefix. - """ - dol = '$' if dollar else '' - if docs := param_doc.get(x): - s = f'{dol}{x}' - return link(doc(tag(s, 'var'), docs), f'#{x}') - else: - return link(tag(f'{dol}{x}', 'var'), f'#{x}') - - -def declare_var(x: str) -> Tag: - """Returna a tag declaring that this variable exists.""" - return tag(id(f"${x}", x), 'var') - - -# TODO strip leading colons when looking up documentation - - -def handle_case_body(forms: list[dict[str, Any]], - indent: int, context: Context) -> Tag: - """Handle case body when parsing AST.""" - ret: list[Markup] = [] - for form in forms: - when = form['when'] - then = form['then'] - ret += [ind(indent+1)] - # cases = [] - - for sublist in intersperse([',', ' '], - [[parse(item, indent+1, context)] - for item in when]): - ret += sublist - - ret += [':', ' ', '{', '\n'] - - for item in then: - ret += [ind(indent+2), parse(item, indent+2, context), '\n'] - ret += [ind(indent+1), '}', '\n'] - - return tag(ret) - - -# Hyperlinks for -# - qn -# - qr -# - var (except when it's the var declaration) - -LineFragment: TypeAlias = str | Markup -Line: TypeAlias = list[LineFragment] - - -def parse_access(how: Any, args: list[Any], *, indent: int, context: list[str]) -> Tag: - """Parse access form.""" - # TODO newlines? - items = [] - items += [parse(how, indent, context), '['] - for sublist in intersperse([',', ' '], - [[parse(arg, indent, context)] - for arg in args]): - items += sublist - items += [']'] - return tag(items, 'access') - - -def parse_array(items: list[Any], *, indent: int, context: list[str]) -> Tag: - """Parse array form.""" - out: list[Markup] - out = ['[', '\n'] - for item in items: - out += [ - ind(indent+2), - parse(item, indent+1, context), - ',', - '\n', - ] - out += [ind(indent), ']'] - return tag(out, 'array') - - -def parse_call(func: Any, args: list[Any], *, indent: int, context: list[str]) -> Tag: - """Parse call form.""" - items = [] - items += [parse(func, indent, context), '('] - for sublist in intersperse([',', ' '], - [[parse(arg, indent, context)] - for arg in args]): - items += sublist - items += [')'] - return tag(items, 'call') - - -def parse_call_method(func: Any, *, indent: int, context: list[str]) -> Tag: - """Parse call method form.""" - items = [parse(func['functor'], indent, context)] - - if not ('block' in func and func['args'] == []): - items += ['('] - for sublist in intersperse([',', ' '], - [[parse(x, indent, context)] - for x in func['args']]): - items += sublist - items += [')'] - - if 'block' in func: - items += [parse(func['block'], indent+1, context)] - - return tag(items, 'call-method') - - -def parse_case(test: Any, forms: Any, *, indent: int, context: list[str]) -> Tag: - """Parse case form.""" - items: list[Markup] = [ - keyword('case'), - ' ', - parse(test, indent, context), - ' ', '{', '\n', - handle_case_body(forms, indent, context), - ind(indent), - '}', - ] - - return tag(items) - - -def parse_class(name: Any, rest: dict[str, Any], - *, indent: int, context: list[str]) -> Tag: - """Parse class form.""" - items: list[Markup] = [] - items += [ - keyword('class'), - ' ', - tag(name, 'name'), - ' ', - ] - - if 'params' in rest: - items += ['(', '\n'] - for name, data in rest['params'].items(): - decls: list[Markup] = [] - decls += [ind(indent+1)] - if 'type' in data: - tt = parse(data['type'], indent+1, context) - decls += [tag(tt, 'type'), - ' '] - decls += [declare_var(name)] - if 'value' in data: - decls += [ - ' ', operator('='), ' ', - # TODO this is a declaration - parse(data.get('value'), indent+1, context), - ] - items += [declaration(decls, 'declaration', variable=name)] - items += [',', '\n'] - items += [ind(indent), ')', ' ', '{', '\n'] - else: - items += ['{', '\n'] - - if 'body' in rest: - for entry in rest['body']: - items += [ind(indent+1), - parse(entry, indent+1, context), - '\n'] - items += [ind(indent), '}'] - return tag(items) - - -def parse_concat(args: list[Any], *, indent: int, context: list[str]) -> Tag: - """Parse concat form.""" - items = ['"'] - for item in args: - match item: - case ['str', ['var', x]]: - items += [tag(['${', print_var(x, False), '}'], 'str-var')] - case ['str', thingy]: - content = parse(thingy, indent, ['str'] + context) - items += [tag(['${', content, '}'], 'str-var')] - case s: - items += [s - .replace('"', '\\"') - .replace('\n', '\\n')] - items += '"' - return tag(items, 'string') - - -def parse_define(name: Any, rest: dict[str, Any], - *, indent: int, context: list[str]) -> Tag: - """Parse define form.""" - items: list[Markup] = [] - items += [keyword('define'), - ' ', - tag(name, 'name'), - ' '] - - if params := rest.get('params'): - items += ['(', '\n'] - for name, data in params.items(): - decl: list[Markup] = [] - decl += [ind(indent+1)] - if 'type' in data: - decl += [tag(parse(data['type'], indent, context), - 'type'), - ' '] - # print(f'<span class="var">${name}</span>', end='') - decl += [declare_var(name)] - if 'value' in data: - decl += [ - ' ', '=', ' ', - parse(data.get('value'), indent, context), - ] - items += [declaration(decl, 'declaration', variable=name)] - items += [',', '\n'] - - items += [ind(indent), ')', ' '] - - items += ['{', '\n'] - - if 'body' in rest: - for entry in rest['body']: - items += [ind(indent+1), - parse(entry, indent+1, context), - '\n'] - - items += [ind(indent), '}'] - - return tag(items) - - -def parse_function(name: Any, rest: dict[str, Any], - *, indent: int, context: list[str]) -> Tag: - """Parse function form.""" - items = [] - items += [keyword('function'), - ' ', name] - if 'params' in rest: - items += [' ', '(', '\n'] - for name, attributes in rest['params'].items(): - items += [ind(indent+1)] - if 'type' in attributes: - items += [parse(attributes['type'], indent, context), - ' '] - items += [f'${name}'] - if 'value' in attributes: - items += [ - ' ', '=', ' ', - parse(attributes['value'], indent, context), - ] - items += [',', '\n'] - items += [ind(indent), ')'] - - if 'returns' in rest: - items += [' ', '>>', ' ', - parse(rest['returns'], indent, context)] - - items += [' ', '{'] - if 'body' in rest: - items += ['\n'] - for item in rest['body']: - items += [ - ind(indent+1), - parse(item, indent+1, context), - '\n', - ] - items += [ind(indent)] - items += ['}'] - return tag(items) - - -def parse_heredoc_concat(parts: list[Any], - *, indent: int, context: list[str]) -> Tag: - """Parse heredoc form containing concatenation.""" - items: list[Markup] = ['@("EOF")'] - - lines: list[Line] = [[]] - - for part in parts: - match part: - case ['str', ['var', x]]: - lines[-1] += [tag(['${', print_var(x, False), '}'])] - case ['str', form]: - lines[-1] += [tag(['${', parse(form, indent, context), '}'])] - case s: - if not isinstance(s, str): - raise ValueError('Unexpected value in heredoc', s) - - first, *rest = s.split('\n') - lines[-1] += [first] - # lines += [[]] - - for item1 in rest: - lines += [[item1]] - - for line in lines: - items += ['\n'] - if line != ['']: - items += [ind(indent)] - for item2 in line: - if item2: - items += [item2] - - match lines: - case [*_, ['']]: - # We have a trailing newline - items += [ind(indent), '|'] - case _: - # We don't have a trailing newline - # Print the graphical one, but add the dash to the pipe - items += ['\n', ind(indent), '|-'] - - items += [' ', 'EOF'] - return tag(items, 'heredoc', 'literal') - - -def parse_heredoc_text(text: str, *, indent: int, context: list[str]) -> Tag: - """Parse heredoc form only containing text.""" - items: list[Markup] = [] - items += ['@(EOF)', '\n'] - lines = text.split('\n') - - no_eol: bool = True - - if lines[-1] == '': - lines = lines[:-1] - no_eol = False - - for line in lines: - if line: - items += [ind(indent), line] - items += ['\n'] - items += [ind(indent)] - - if no_eol: - items += ['|-'] - else: - items += ['|'] - items += [' ', 'EOF'] - - return tag(items, 'heredoc', 'literal') - - -def parse_if(test: Any, rest: dict[str, Any], *, indent: int, context: list[str]) -> Tag: - """Parse if form.""" - items: list[Markup] = [] - items += [ - keyword('if'), - ' ', - parse(test, indent, context), - ' ', '{', '\n', - ] - if 'then' in rest: - for item in rest['then']: - items += [ - ind(indent+1), - parse(item, indent+1, context), - '\n', - ] - items += [ind(indent), '}'] - - if 'else' in rest: - items += [' '] - match rest['else']: - case [['if', *rest]]: - # TODO propper tagging - items += ['els', - parse(['if', *rest], indent, context)] - case el: - items += [keyword('else'), - ' ', '{', '\n'] - for item in el: - items += [ - ind(indent+1), - parse(item, indent+1, context), - '\n', - ] - items += [ - ind(indent), - '}', - ] - return tag(items) - - -def parse_invoke(func: Any, args: list[Any], - *, indent: int, context: list[str]) -> Tag: - """Parse invoke form.""" - items = [ - parse(func, indent, context), - ' ', - ] - if len(args) == 1: - items += [parse(args[0], indent+1, context)] - else: - items += ['('] - for sublist in intersperse([',', ' '], - [[parse(arg, indent+1, context)] - for arg in args]): - items += sublist - items += [')'] - return tag(items, 'invoke') - - -def parse_lambda(params: dict[str, Any], body: Any, - *, indent: int, context: list[str]) -> Tag: - """Parse lambda form.""" - items: list[Markup] = [] - # TODO note these are declarations - items += ['|'] - for sublist in intersperse([',', ' '], - [[f'${x}'] for x in params.keys()]): - items += sublist - items += ['|', ' ', '{', '\n'] - for entry in body: - items += [ - ind(indent), - parse(entry, indent, context), - '\n', - ] - items += [ind(indent-1), '}'] - return tag(items, 'lambda') - - -def parse_resource(t: str, bodies: list[Any], - *, indent: int, context: list[str]) -> Tag: - """Parse resource form.""" - match bodies: - case [body]: - items = [ - parse(t, indent, context), - ' ', '{', ' ', - parse(body['title'], indent, context), - ':', '\n', - ] - ops = body['ops'] - - namelen = ops_namelen(ops) - - for item in ops: - match item: - case ['=>', key, value]: - pad = namelen - len(key) - items += [ - ind(indent+1), - tag(key, 'parameter'), - ' '*pad, ' ', '=>', ' ', - parse(value, indent+1, context), - ',', '\n', - ] - - case ['splat-hash', value]: - items += [ - ind(indent+1), - tag('*', 'parameter', 'splat'), - ' '*(namelen-1), - ' ', '=>', ' ', - parse(value, indent+1, context), - ',', '\n', - ] - - case _: - raise Exception("Unexpected item in resource:", item) - - items += [ - ind(indent), - '}', - ] - - return tag(items) - case bodies: - items = [] - items += [ - parse(t, indent, context), - ' ', '{', - ] - for body in bodies: - items += [ - '\n', ind(indent+1), - parse(body['title'], indent, context), - ':', '\n', - ] - - ops = body['ops'] - namelen = ops_namelen(ops) - - for item in ops: - match item: - case ['=>', key, value]: - pad = namelen - len(key) - items += [ - ind(indent+2), - tag(key, 'parameter'), - ' '*pad, - ' ', '=>', ' ', - parse(value, indent+2, context), - ',', '\n', - ] - - case ['splat-hash', value]: - items += [ - ind(indent+2), - tag('*', 'parameter', 'splat'), - ' '*(namelen - 1), - ' ', '=>', ' ', - parse(value, indent+2, context), - ',', '\n', - ] - - case _: - raise Exception("Unexpected item in resource:", item) - - items += [ind(indent+1), ';'] - items += ['\n', ind(indent), '}'] - return tag(items) - - -def parse_resource_defaults(t: str, ops: Any, - *, indent: int, context: list[str]) -> Tag: - """Parse resource defaults form.""" - items = [ - parse(t, indent, context), - ' ', '{', '\n', - ] - namelen = ops_namelen(ops) - for op in ops: - match op: - case ['=>', key, value]: - pad = namelen - len(key) - items += [ - ind(indent+1), - tag(key, 'parameter'), - ' '*pad, - ' ', operator('=>'), ' ', - parse(value, indent+3, context), - ',', '\n', - ] - - case ['splat-hash', value]: - pad = namelen - 1 - items += [ - ind(indent+1), - tag('*', 'parameter', 'splat'), - ' '*pad, - ' ', operator('=>'), ' ', - parse(value, indent+2, context), - ',', '\n', - ] - - case x: - raise Exception('Unexpected item in resource defaults:', x) - - items += [ind(indent), - '}'] - - return tag(items) - - -def parse_resource_override(resources: Any, ops: Any, - *, indent: int, context: list[str]) -> Tag: - """Parse resoruce override form.""" - items = [ - parse(resources, indent, context), - ' ', '{', '\n', - ] - - namelen = ops_namelen(ops) - for op in ops: - match op: - case ['=>', key, value]: - pad = namelen - len(key) - items += [ - ind(indent+1), - tag(key, 'parameter'), - ' '*pad, - ' ', operator('=>'), ' ', - parse(value, indent+3, context), - ',', '\n', - ] - - case ['+>', key, value]: - pad = namelen - len(key) - items += [ - ind(indent+1), - tag(key, 'parameter'), - ' '*pad, - ' ', operator('+>'), ' ', - parse(value, indent+2, context), - ',', '\n', - ] - - case ['splat-hash', value]: - pad = namelen - 1 - items += [ - ind(indent+1), - tag('*', 'parameter', 'splat'), - ' '*pad, - ' ', operator('=>'), ' ', - parse(value, indent+2, context), - ',', '\n', - ] - - case _: - raise Exception('Unexpected item in resource override:', - op) - - items += [ - ind(indent), - '}', - ] - - return tag(items) - - -def parse_unless(test: Any, rest: dict[str, Any], - *, indent: int, context: list[str]) -> Tag: - """Parse unless form.""" - items: list[Markup] = [ - keyword('unless'), - ' ', - parse(test, indent, context), - ' ', '{', '\n', - ] - - if 'then' in rest: - for item in rest['then']: - items += [ - ind(indent+1), - parse(item, indent+1, context), - '\n', - ] - - items += [ - ind(indent), - '}', - ] - return tag(items) - - -def parse_operator(op: str, lhs: Any, rhs: Any, - *, indent: int, context: list[str]) -> Tag: - """Parse binary generic operator form.""" - return tag([ - parse(lhs, indent, context), - ' ', operator(op), ' ', - parse(rhs, indent, context), - ]) - - -def parse(form: Any, indent: int, context: list[str]) -> Markup: - """ - Print everything from a puppet parse tree. - - :param from: - A puppet AST. - :param indent: - How many levels deep in indentation the code is. - Will get multiplied by the indentation width. - """ - items: list[Markup] - # Sorted per `sort -V` - match form: - case None: - return tag('undef', 'literal', 'undef') - - case True: - return tag('true', 'literal', 'true') - - case False: - return tag('false', 'literal', 'false') - - case ['access', how, *args]: - return parse_access(how, args, indent=indent, context=context) - - case ['and', a, b]: - return tag([ - parse(a, indent, context), - ' ', keyword('and'), ' ', - parse(b, indent, context), - ]) - - case ['array']: - return tag('[]', 'array') - - case ['array', *items]: - return parse_array(items, indent=indent, context=context) - - case ['call', {'functor': func, 'args': args}]: - return parse_call(func, args, indent=indent, context=context) - - case ['call-method', func]: - return parse_call_method(func, indent=indent, context=context) - - case ['case', test, forms]: - return parse_case(test, forms, indent=indent, context=context) - - case ['class', {'name': name, **rest}]: - return parse_class(name, rest, indent=indent, context=context) - - case ['concat', *args]: - return parse_concat(args, indent=indent, context=context) - - case ['collect', {'type': t, 'query': q}]: - return tag([parse(t, indent, context), - ' ', - parse(q, indent, context)]) - - case ['default']: - return keyword('default') - - case ['define', {'name': name, **rest}]: - return parse_define(name, rest, indent=indent, context=context) - - case ['exported-query']: - return tag(['<<|', ' ', '|>>']) - - case ['exported-query', arg]: - return tag(['<<|', ' ', parse(arg, indent, context), ' ', '|>>']) - - case ['function', {'name': name, **rest}]: - return parse_function(name, rest, indent=indent, context=context) - - case ['hash']: - return tag('{}', 'hash') - - case ['hash', *hash]: - return tag([ - '{', '\n', - print_hash(hash, indent+1, context), - ind(indent), - '}', - ], 'hash') - - # TODO a safe string to use? - # TODO extra options? - # Are all these already removed by the parser, requiring - # us to reverse parse the text? - - # Parts can NEVER be empty, since that case wouldn't generate - # a concat element, but a "plain" text element - case ['heredoc', {'text': ['concat', *parts]}]: - return parse_heredoc_concat(parts, indent=indent, context=context) - - case ['heredoc', {'text': ''}]: - return tag(['@(EOF)', '\n', ind(indent), '|', ' ', 'EOF'], - 'heredoc', 'literal') - - case ['heredoc', {'text': text}]: - return parse_heredoc_text(text, indent=indent, context=context) - - case ['if', {'test': test, **rest}]: - return parse_if(test, rest, indent=indent, context=context) - - case ['in', needle, stack]: - return tag([ - parse(needle, indent, context), - ' ', keyword('in'), ' ', - parse(stack, indent, context), - ]) - - case ['invoke', {'functor': func, 'args': args}]: - return parse_invoke(func, args, indent=indent, context=context) - - case ['nop']: - return tag('', 'nop') - - case ['lambda', {'params': params, 'body': body}]: - return parse_lambda(params, body, indent=indent, context=context) - - case ['or', a, b]: - return tag([ - parse(a, indent, context), - ' ', keyword('or'), ' ', - parse(b, indent, context), - ]) - - case ['paren', *forms]: - return tag([ - '(', - *(parse(form, indent+1, context) - for form in forms), - ')', - ], 'paren') - - # Qualified name? - case ['qn', x]: - return tag(x, 'qn') - - # Qualified resource? - case ['qr', x]: - return tag(x, 'qr') - - case ['regexp', s]: - return tag(['/', tag(s, 'regex-body'), '/'], 'regex') - - # Resource instansiation with exactly one instance - case ['resource', {'type': t, 'bodies': [body]}]: - return parse_resource(t, [body], indent=indent, context=context) - - # Resource instansiation with any number of instances - case ['resource', {'type': t, 'bodies': bodies}]: - return parse_resource(t, bodies, indent=indent, context=context) - - case ['resource-defaults', {'type': t, 'ops': ops}]: - return parse_resource_defaults(t, ops, indent=indent, context=context) - - case ['resource-override', {'resources': resources, 'ops': ops}]: - return parse_resource_override(resources, ops, indent=indent, context=context) - - case ['unless', {'test': test, **rest}]: - return parse_unless(test, rest, indent=indent, context=context) - - case ['var', x]: - if context[0] == 'declaration': - return declare_var(x) - else: - return print_var(x, True) - - case ['virtual-query', q]: - return tag(['<|', ' ', parse(q, indent, context), ' ', '|>', ]) - - case ['virtual-query']: - return tag(['<|', ' ', '|>']) - - case ['!', x]: - return tag([ - operator('!'), ' ', - parse(x, indent, context), - ]) - - case ['-', a]: - return tag([ - operator('-'), ' ', - parse(a, indent, context), - ]) - - case [('!=' | '+' | '-' | '*' | '%' | '<<' | '>>' | '>=' | '<=' | '>' | '<' | '/' | '==' | '=~' | '!~') as op, # noqa: E501 - a, b]: - return parse_operator(op, a, b, indent=indent, context=context) - - case ['~>', left, right]: - return tag([ - parse(left, indent, context), - '\n', - ind(indent), - operator('~>'), ' ', - parse(right, indent, context) - ]) - - case ['->', left, right]: - return tag([ - parse(left, indent, context), - '\n', - ind(indent), - operator('->'), ' ', - parse(right, indent, context), - ]) - - case ['.', left, right]: - return tag([ - parse(left, indent, context), - '\n', - ind(indent), - operator('.'), - parse(right, indent+1, context), - ]) - - case ['=', field, value]: - return tag([ - parse(field, indent, ['declaration'] + context), - ' ', operator('='), ' ', - parse(value, indent, context), - ], 'declaration') - - case ['?', condition, cases]: - return tag([ - parse(condition, indent, context), - ' ', operator('?'), ' ', '{', '\n', - print_hash(cases, indent+1, context), - ind(indent), - '}', - ], 'case') - - case form: - if isinstance(form, str): - # TODO remove this case? - # if context[0] == 'heredoc': - # lines: list[str] - # match form.split('\n'): - # case [*_lines, '']: - # lines = _lines - # case _lines: - # lines = _lines - - # items = [] - # for line in lines: - # items += [ind(indent), line, '\n'] - - # return tag(items, 'literal', 'string') - - # else: - # TODO further escaping? - s = form.replace('\n', r'\n') - s = f"'{s}'" - return tag(s, 'literal', 'string') - - elif isinstance(form, int) or isinstance(form, float): - return tag(str(form), 'literal', 'number') - else: - return tag(f'[|[{form}]|]', 'parse-error') +# -------------------------------------------------- def format_docstring(name: str, docstring: DocString) -> Tuple[str, str]: @@ -1114,17 +125,22 @@ def build_param_dict(docstring: DocString) -> dict[str, str]: def format_class(d_type: DefinedType | PuppetClass) -> Tuple[str, str]: """Format Puppet class.""" - t = parse_puppet(d_type.source) - data = parse(t, 0, ['root']) - renderer = HTMLRenderer(build_param_dict(d_type.docstring)) out = '' name = d_type.name + logger.debug("Formatting class %s", name) # print(name, file=sys.stderr) name, body = format_docstring(name, d_type.docstring) out += body out += '<pre class="highlight-muppet"><code class="puppet">' - out += render(renderer, data) + # ------ Old --------------------------------------- + # t = parse_puppet(d_type.source) + # data = parse(t, 0, ['root']) + # renderer = HTMLRenderer(build_param_dict(d_type.docstring)) + # out += render(renderer, data) + # ------ New --------------------------------------- + ast = build_ast(puppet_parser(d_type.source)) + out += serialize(ast, HTMLFormatter) out += '</code></pre>' return name, out @@ -1136,33 +152,33 @@ def format_type() -> str: def format_type_alias(d_type: DataTypeAlias) -> Tuple[str, str]: """Format Puppet type alias.""" - renderer = HTMLRenderer() out = '' name = d_type.name + logger.debug("Formatting type alias %s", name) # print(name, file=sys.stderr) title, body = format_docstring(name, d_type.docstring) out += body out += '\n' out += '<pre class="highlight-muppet"><code class="puppet">' - t = parse_puppet(d_type.alias_of) - data = parse(t, 0, ['root']) - out += render(renderer, data) + t = puppet_parser(d_type.alias_of) + data = build_ast(t) + out += serialize(data, HTMLFormatter) out += '</code></pre>\n' return title, out def format_defined_type(d_type: DefinedType) -> Tuple[str, str]: """Format Puppet defined type.""" - renderer = HTMLRenderer(build_param_dict(d_type.docstring)) + # renderer = HTMLRenderer(build_param_dict(d_type.docstring)) out = '' name = d_type.name + logger.debug("Formatting defined type %s", name) # print(name, file=sys.stderr) title, body = format_docstring(name, d_type.docstring) out += body out += '<pre class="highlight-muppet"><code class="puppet">' - t = parse_puppet(d_type.source) - out += render(renderer, parse(t, 0, ['root'])) + out += serialize(build_ast(puppet_parser(d_type.source)), HTMLFormatter) out += '</code></pre>\n' return title, out @@ -1170,6 +186,7 @@ def format_defined_type(d_type: DefinedType) -> Tuple[str, str]: def format_resource_type(r_type: ResourceType) -> str: """Format Puppet resource type.""" name = r_type.name + logger.debug("Formatting resource type %s", name) out = '' out += f'<h2>{name}</h2>\n' out += str(r_type.docstring) @@ -1207,6 +224,7 @@ def format_puppet_function(function: Function) -> str: """Format Puppet function.""" out = '' name = function.name + logger.debug("Formatting puppet function %s", name) out += f'<h2>{name}</h2>\n' t = function.type # docstring = function.docstring @@ -1222,8 +240,9 @@ def format_puppet_function(function: Function) -> str: elif t == 'puppet': out += '<pre class="highlight-muppet"><code class="puppet">' try: - source = parse_puppet(function.source) - out += str(parse(source, 0, ['root'])) + # source = parse_puppet(function.source) + # out += str(build_ast(source)) + out += serialize(build_ast(puppet_parser(function.source)), HTMLFormatter) except CalledProcessError as e: print(e, file=sys.stderr) print(f"Failed on function: {name}", file=sys.stderr) diff --git a/muppet/puppet/__main__.py b/muppet/puppet/__main__.py new file mode 100644 index 0000000..080a2db --- /dev/null +++ b/muppet/puppet/__main__.py @@ -0,0 +1,72 @@ +"""Extra executable for testing the various parser steps.""" + +from .parser import puppet_parser +from .ast import build_ast +from .format.text import TextFormatter +from .format import serialize +import json +import argparse +import pprint +import sys +from typing import Any + + +def __main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('mode', + choices=['raw', 'parser', 'ast', 'serialize'], + help='Mode of operation') + + parser.add_argument('file', + type=argparse.FileType('r'), + help='Puppet file to parse') + parser.add_argument('--format', + choices=['json', 'python'], + action='store', + default='json', + help='Format of the output') + parser.add_argument('--pretty', + nargs='?', + action='store', + type=int, + help='Prettify the output, optionally specifying indent width') + + args = parser.parse_args() + + def output(thingy: Any) -> None: + match args.format: + case 'json': + json.dump(result, sys.stdout, indent=args.pretty) + print() + case 'python': + if indent := args.pretty: + pprint.pprint(thingy, indent=indent) + else: + print(thingy) + + data = args.file.read() + if args.mode == 'raw': + output(data) + return + + # print("raw data:", data) + result = puppet_parser(data) + if args.mode == 'parser': + output(result) + return + + ast = build_ast(result) + if args.mode == 'ast': + output(ast) + return + + reserialized = serialize(ast, TextFormatter) + if args.mode == 'serialize': + print(reserialized) + return + + print('No mode given') + + +if __name__ == '__main__': + __main() diff --git a/muppet/puppet/ast.py b/muppet/puppet/ast.py new file mode 100644 index 0000000..2ce3c35 --- /dev/null +++ b/muppet/puppet/ast.py @@ -0,0 +1,724 @@ +"""The Puppet AST, in Python.""" + +from dataclasses import dataclass, field +import logging + +from typing import ( + Any, + Literal, + Optional, +) +# from abc import abstractmethod + + +logger = logging.getLogger(__name__) + + +# -------------------------------------------------- + + +@dataclass +class HashEntry: + """An entry in a hash table.""" + + k: 'Puppet' + v: 'Puppet' + + +@dataclass +class PuppetDeclarationParameter: + """ + A parameter to a class, definition, or function. + + .. code-block:: puppet + + class A ( + type k = v, + ) { + } + """ + + k: str + v: Optional['Puppet'] = None + type: Optional['Puppet'] = None + + +# -------------------------------------------------- + +@dataclass(kw_only=True) +class Puppet: + """Base for puppet item.""" + + +@dataclass +class PuppetLiteral(Puppet): + """ + A self quoting value. + + e.g. ``true``, ``undef``, ... + """ + + literal: str + + +@dataclass +class PuppetAccess(Puppet): + """ + Array access and similar. + + .. code-block:: puppet + + how[args] + """ + + how: Puppet + args: list[Puppet] + + +@dataclass +class PuppetBinaryOperator(Puppet): + """An operator with two values.""" + + op: str + lhs: Puppet + rhs: Puppet + + +@dataclass +class PuppetUnaryOperator(Puppet): + """A prefix operator.""" + + op: str + x: Puppet + + +@dataclass +class PuppetArray(Puppet): + """An array of values.""" + + items: list[Puppet] + + +@dataclass +class PuppetCall(Puppet): + """ + A function call. + + .. highlight: puppet + + func(args) + """ + + func: Puppet + args: list[Puppet] + + +@dataclass +class PuppetCallMethod(Puppet): + """A method call? TODO.""" + + func: Puppet + args: list[Puppet] + block: Optional[Puppet] = None + + +@dataclass +class PuppetCase(Puppet): + """A case "statement".""" + + test: Puppet + cases: list[tuple[list[Puppet], list[Puppet]]] + + +@dataclass +class PuppetInstanciationParameter(Puppet): + """ + Key-value pair used when instanciating resources. + + Also used with resource defaults and resoruce overrides. + + ``+>`` arrow is only valid on overrides, + `See <https://www.puppet.com/docs/puppet/7/lang_resources.html>`. + + .. code-block:: puppet + + file { 'hello': + k => v, + } + """ + + k: str + v: Puppet + arrow: Literal['=>'] | Literal['+>'] = '=>' + + +@dataclass +class PuppetClass(Puppet): + """A puppet class declaration.""" + + name: str + params: Optional[list[PuppetDeclarationParameter]] = None + body: list[Puppet] = field(default_factory=list) + + +@dataclass +class PuppetConcat(Puppet): + """A string with interpolated values.""" + + fragments: list[Puppet] + + +@dataclass +class PuppetCollect(Puppet): + """ + Resource collectors. + + These should be followed by a query (either exported or virtual). + """ + + type: Puppet + query: Puppet + + +@dataclass +class PuppetIf(Puppet): + """ + A puppet if expression. + + .. code-block:: puppet + + if condition { + consequent + } else { + alretnative + } + + ``elsif`` is parsed as an else block with a single if expression + in it. + """ + + condition: Puppet + consequent: list[Puppet] + alternative: Optional[list[Puppet]] = None + + +@dataclass +class PuppetUnless(Puppet): + """A puppet unless expression.""" + + condition: Puppet + consequent: list[Puppet] + + +@dataclass +class PuppetKeyword(Puppet): + """ + A reserved word in the puppet language. + + This class is seldom instanciated, since most keywords are + embedded in the other forms. + """ + + name: str + + +@dataclass +class PuppetExportedQuery(Puppet): + """ + An exported query. + + .. highlight:: puppet + + <<| filter |>> + """ + + filter: Optional[Puppet] = None + + +@dataclass +class PuppetVirtualQuery(Puppet): + """ + A virtual query. + + .. highlight:: puppet + + <| q |> + """ + + q: Optional[Puppet] = None + + +@dataclass +class PuppetFunction(Puppet): + """Declaration of a Puppet function.""" + + name: str + params: Optional[list[PuppetDeclarationParameter]] = None + returns: Optional[Puppet] = None + body: list[Puppet] = field(default_factory=list) + + +@dataclass +class PuppetHash(Puppet): + """A puppet dictionary.""" + + entries: list[HashEntry] = field(default_factory=list) + + +@dataclass +class PuppetHeredoc(Puppet): + """A puppet heredoc.""" + + fragments: list[Puppet] + syntax: Optional[str] = None + + +@dataclass +class PuppetLiteralHeredoc(Puppet): + """A puppet heredoc without any interpolation.""" + + content: str + syntax: Optional[str] = None + + +@dataclass +class PuppetVar(Puppet): + """A puppet variable.""" + + name: str + + +@dataclass +class PuppetLambda(Puppet): + """A puppet lambda.""" + + params: list[PuppetDeclarationParameter] + body: list[Puppet] + + +@dataclass +class PuppetParenthesis(Puppet): + """Forms surrounded by parethesis.""" + + form: Puppet + + +@dataclass +class PuppetQn(Puppet): + """Qn TODO.""" + + name: str + + +@dataclass +class PuppetQr(Puppet): + """Qr TODO.""" + + name: str + + +@dataclass +class PuppetRegex(Puppet): + """A regex literal.""" + + s: str + + +@dataclass +class PuppetResource(Puppet): + """Instansiation of one (or more) puppet resources.""" + + type: Puppet + bodies: list[tuple[Puppet, list[PuppetInstanciationParameter]]] + + +@dataclass +class PuppetNop(Puppet): + """A no-op.""" + + def serialize(self, indent: int, **_: Any) -> str: # noqa: D102 + return '' + + +@dataclass +class PuppetDefine(Puppet): + """A puppet resource declaration.""" + + name: str + params: Optional[list[PuppetDeclarationParameter]] = None + body: list[Puppet] = field(default_factory=list) + + +@dataclass +class PuppetString(Puppet): + """A puppet string literal.""" + + s: str + + +@dataclass +class PuppetNumber(Puppet): + """A puppet numeric literal.""" + + x: int | float + + +@dataclass +class PuppetInvoke(Puppet): + """ + A puppet function invocation. + + This is at least used when including classes: + + .. code-block:: puppet + + include ::example + + Where func is ``include``, and args is ``[::example]`` + """ + + func: Puppet + args: list[Puppet] + + +@dataclass +class PuppetResourceDefaults(Puppet): + """ + Default values for a resource. + + .. code-block:: puppet + + File { + x => 10, + } + """ + + type: Puppet + ops: list[PuppetInstanciationParameter] + + +@dataclass +class PuppetResourceOverride(Puppet): + """ + A resource override. + + .. code-block:: puppet + + File["name"] { + x => 10, + } + """ + + resource: Puppet + ops: list[PuppetInstanciationParameter] + + +@dataclass +class PuppetDeclaration(Puppet): + """A stand-alone variable declaration.""" + + k: Puppet # PuppetVar + v: Puppet + + +@dataclass +class PuppetSelector(Puppet): + """ + A puppet selector. + + .. code-block:: puppet + + resource ? { + case_match => case_body, + } + """ + + resource: Puppet + cases: list[tuple[Puppet, Puppet]] + + +@dataclass +class PuppetBlock(Puppet): + """Used if multiple top level items exists.""" + + entries: list[Puppet] + + +@dataclass +class PuppetNode(Puppet): + """ + A node declaration. + + Used with an ENC is not in use, + + .. code-block:: puppet + + node 'host.example.com' { + include ::profiles::example + } + """ + + matches: list[Puppet] + body: list[Puppet] + + +@dataclass +class PuppetParseError(Puppet): + """Anything we don't know how to handle.""" + + x: Any = None + + def serialize(self, indent: int, **_: Any) -> str: # noqa: D102 + return f'INVALID INPUT: {repr(self.x)}' + +# ---------------------------------------------------------------------- + + +def parse_hash_entry(data: list[Any]) -> HashEntry: + """Parse a single hash entry.""" + match data: + case [_, key, value]: + return HashEntry(k=build_ast(key), v=build_ast(value)) + case _: + raise ValueError(f'Not a hash entry {data}') + + +def parse_puppet_declaration_params(data: dict[str, dict[str, Any]]) \ + -> list[PuppetDeclarationParameter]: + """Parse a complete set of parameters.""" + parameters = [] + + for name, definition in data.items(): + type: Optional[Puppet] = None + value: Optional[Puppet] = None + if t := definition.get('type'): + type = build_ast(t) + if v := definition.get('value'): + value = build_ast(v) + + parameters.append(PuppetDeclarationParameter(k=name, v=value, type=type)) + + return parameters + + +def parse_puppet_instanciation_param(data: list[Any]) -> PuppetInstanciationParameter: + """Parse a single parameter.""" + match data: + case [arrow, key, value]: + return PuppetInstanciationParameter( + k=key, v=build_ast(value), arrow=arrow) + case ['splat-hash', value]: + return PuppetInstanciationParameter('*', build_ast(value)) + case _: + raise ValueError(f'Not an instanciation parameter {data}') + + +# ---------------------------------------------------------------------- + + +def build_ast(form: Any) -> Puppet: + """ + Print everything from a puppet parse tree. + + :param from: + A puppet AST. + :param indent: + How many levels deep in indentation the code is. + Will get multiplied by the indentation width. + """ + # items: list[Markup] + match form: + case None: return PuppetLiteral('undef') # noqa: E272 + case True: return PuppetLiteral('true') # noqa: E272 + case False: return PuppetLiteral('false') + case ['default']: return PuppetKeyword('default') + + case ['access', how, *args]: + return PuppetAccess(how=build_ast(how), + args=[build_ast(arg) for arg in args], + ) + + case [('and' | 'or' | 'in' | '!=' | '+' | '-' | '*' | '%' + | '<<' | '>>' | '>=' | '<=' | '>' | '<' | '/' | '==' + | '=~' | '!~' | '~>' | '->' | '.') as op, + a, b]: + return PuppetBinaryOperator( + op=op, + lhs=build_ast(a), + rhs=build_ast(b), + ) + + case [('!' | '-') as op, x]: + return PuppetUnaryOperator(op=op, x=build_ast(x)) + + case ['array', *items]: + return PuppetArray([build_ast(x) for x in items]) + + case ['call', {'functor': func, 'args': args}]: + return PuppetCall( + build_ast(func), + [build_ast(x) for x in args]) + + case ['call-method', {'functor': func, 'args': args, 'block': block}]: + return PuppetCallMethod(func=build_ast(func), + args=[build_ast(x) for x in args], + block=build_ast(block)) + + case ['call-method', {'functor': func, 'args': args}]: + return PuppetCallMethod(func=build_ast(func), + args=[build_ast(x) for x in args]) + + case ['case', test, forms]: + cases = [] + for form in forms: + cases.append(( + [build_ast(x) for x in form['when']], + [build_ast(x) for x in form['then']])) + + return PuppetCase(build_ast(test), cases) + + case [('class' | 'define' | 'function') as what, {'name': name, **rest}]: + cls = { + 'class': PuppetClass, + 'define': PuppetDefine, + 'function': PuppetFunction, + }[what] + + args = {'name': name} + + if p := rest.get('params'): + args['params'] = parse_puppet_declaration_params(p) + + if b := rest.get('body'): + args['body'] = [build_ast(x) for x in b] + + # This is only valid for 'function', and simply will + # never be true for the other cases. + if r := rest.get('returns'): + args['returns'] = build_ast(r) + + return cls(**args) + + case ['concat', *parts]: + return PuppetConcat([build_ast(p) for p in parts]) + + case ['collect', {'type': t, 'query': q}]: + return PuppetCollect(build_ast(t), + build_ast(q)) + + case ['exported-query']: + return PuppetExportedQuery() + + case ['exported-query', arg]: + return PuppetExportedQuery(build_ast(arg)) + + case ['hash']: + return PuppetHash() + case ['hash', *hash]: + return PuppetHash([parse_hash_entry(e) for e in hash]) + + # Parts can NEVER be empty, since that case wouldn't generate + # a concat element, but a "plain" text element + case ['heredoc', {'syntax': syntax, 'text': ['concat', *parts]}]: + return PuppetHeredoc([build_ast(p) for p in parts], syntax=syntax) + + case ['heredoc', {'text': ['concat', *parts]}]: + return PuppetHeredoc([build_ast(p) for p in parts]) + + case ['heredoc', {'syntax': syntax, 'text': text}]: + return PuppetLiteralHeredoc(text, syntax=syntax) + + case ['heredoc', {'text': text}]: + return PuppetLiteralHeredoc(text) + + # Non-literal part of a string or heredoc with interpolation + case ['str', form]: + return build_ast(form) + + case ['if', {'test': test, **rest}]: + consequent = [] + alternative = None + if then := rest.get('then'): + consequent = [build_ast(x) for x in then] + if els := rest.get('else'): + alternative = [build_ast(x) for x in els] + return PuppetIf(build_ast(test), consequent, alternative) + + case ['unless', {'test': test, 'then': forms}]: + return PuppetUnless(build_ast(test), [build_ast(x) for x in forms]) + case ['unless', {'test': test}]: + return PuppetUnless(build_ast(test), []) + + case ['invoke', {'functor': func, 'args': args}]: + return PuppetInvoke(build_ast(func), [build_ast(x) for x in args]) + + case ['nop']: return PuppetNop() + + case ['lambda', {'params': params, 'body': body}]: + return PuppetLambda(params=parse_puppet_declaration_params(params), + body=[build_ast(x) for x in body]) + + case ['lambda', {'body': body}]: + return PuppetLambda([], [build_ast(x) for x in body]) + + case ['paren', form]: + return PuppetParenthesis(build_ast(form)) + + # Qualified name and Qualified resource? + case ['qn', x]: return PuppetQn(x) + case ['qr', x]: return PuppetQr(x) + + case ['var', x]: return PuppetVar(x) + + case ['regexp', s]: return PuppetRegex(s) + + case ['resource', {'type': t, 'bodies': bodies}]: + return PuppetResource( + build_ast(t), + [(build_ast(body['title']), + [parse_puppet_instanciation_param(x) for x in body['ops']]) + for body in bodies]) + + case ['resource-defaults', {'type': t, 'ops': ops}]: + return PuppetResourceDefaults( + build_ast(t), + [parse_puppet_instanciation_param(x) for x in ops]) + + case ['resource-override', {'resources': resources, 'ops': ops}]: + return PuppetResourceOverride( + build_ast(resources), + [parse_puppet_instanciation_param(x) for x in ops]) + + case ['virtual-query']: return PuppetVirtualQuery() + case ['virtual-query', q]: return PuppetVirtualQuery(build_ast(q)) + + case ['=', field, value]: + return PuppetDeclaration(k=build_ast(field), v=build_ast(value)) + + case ['?', condition, cases]: + cases_ = [] + for case in cases: + match case: + case [_, lhs, rhs]: + cases_.append((build_ast(lhs), + build_ast(rhs))) + case _: + raise ValueError(f"Unexepcted '?' form: {case}") + return PuppetSelector(build_ast(condition), cases_) + + case ['block', *items]: + return PuppetBlock([build_ast(x) for x in items]) + + case ['node', {'matches': matches, 'body': body}]: + return PuppetNode([build_ast(x) for x in matches], + [build_ast(x) for x in body]) + + case str(s): + return PuppetString(s) + + case int(x): return PuppetNumber(x) + case float(x): return PuppetNumber(x) + + case default: + logger.warning("Unhandled item: %a", default) + return PuppetParseError(default) diff --git a/muppet/puppet/format/__init__.py b/muppet/puppet/format/__init__.py new file mode 100644 index 0000000..d40f9b4 --- /dev/null +++ b/muppet/puppet/format/__init__.py @@ -0,0 +1,13 @@ +"""Fromat Puppet AST's into something useful.""" + +from .base import Serializer +from muppet.puppet.ast import Puppet +from typing import TypeVar + + +T = TypeVar('T') + + +def serialize(ast: Puppet, serializer: type[Serializer[T]]) -> T: + """Run the given serializer on the given data.""" + return serializer().serialize(ast, 0) diff --git a/muppet/puppet/format/base.py b/muppet/puppet/format/base.py new file mode 100644 index 0000000..9afad2b --- /dev/null +++ b/muppet/puppet/format/base.py @@ -0,0 +1,296 @@ +"""Base class for serializing AST's into something useful.""" + +from typing import ( + TypeVar, + Generic, + final, +) +import logging + +from muppet.puppet.ast import ( + Puppet, + + PuppetLiteral, PuppetAccess, PuppetBinaryOperator, + PuppetUnaryOperator, PuppetArray, PuppetCallMethod, + PuppetCase, PuppetDeclarationParameter, + PuppetInstanciationParameter, PuppetClass, PuppetConcat, + PuppetCollect, PuppetIf, PuppetUnless, PuppetKeyword, + PuppetExportedQuery, PuppetVirtualQuery, PuppetFunction, + PuppetHash, PuppetHeredoc, PuppetLiteralHeredoc, PuppetVar, + PuppetLambda, PuppetQn, PuppetQr, PuppetRegex, + PuppetResource, PuppetDefine, PuppetString, + PuppetNumber, PuppetInvoke, PuppetResourceDefaults, + PuppetResourceOverride, PuppetDeclaration, PuppetSelector, + PuppetBlock, PuppetNode, + PuppetCall, PuppetParenthesis, PuppetNop, + + # HashEntry, + # PuppetParseError, +) + + +T = TypeVar('T') +logger = logging.getLogger(__name__) + + +class Serializer(Generic[T]): + """ + Base class for serialization. + + the ``serialize`` method dispatches depending on the type of the argument, + and should be called recursively when needed. + + All other methods implement the actual serialization, and MUST be extended + by each instance. + """ + + @classmethod + def _puppet_literal(cls, it: PuppetLiteral, indent: int) -> T: + raise NotImplementedError("puppet_literal must be implemented by subclass") + + @classmethod + def _puppet_access(cls, it: PuppetAccess, indent: int) -> T: + raise NotImplementedError("puppet_access must be implemented by subclass") + + @classmethod + def _puppet_binary_operator(cls, it: PuppetBinaryOperator, indent: int) -> T: + raise NotImplementedError("puppet_binary_operator must be implemented by subclass") + + @classmethod + def _puppet_unary_operator(cls, it: PuppetUnaryOperator, indent: int) -> T: + raise NotImplementedError("puppet_unary_operator must be implemented by subclass") + + @classmethod + def _puppet_array(cls, it: PuppetArray, indent: int) -> T: + raise NotImplementedError("puppet_array must be implemented by subclass") + + @classmethod + def _puppet_call(cls, it: PuppetCall, indent: int) -> T: + raise NotImplementedError("puppet_call must be implemented by subclass") + + @classmethod + def _puppet_call_method(cls, it: PuppetCallMethod, indent: int) -> T: + raise NotImplementedError("puppet_call_method must be implemented by subclass") + + @classmethod + def _puppet_case(cls, it: PuppetCase, indent: int) -> T: + raise NotImplementedError("puppet_case must be implemented by subclass") + + @classmethod + def _puppet_declaration_parameter(cls, it: PuppetDeclarationParameter, indent: int) -> T: + raise NotImplementedError("puppet_declaration_parameter must be implemented by subclass") + + @classmethod + def _puppet_instanciation_parameter(cls, it: PuppetInstanciationParameter, indent: int) -> T: + raise NotImplementedError("puppet_instanciation_parameter must be implemented by subclass") + + @classmethod + def _puppet_class(cls, it: PuppetClass, indent: int) -> T: + raise NotImplementedError("puppet_class must be implemented by subclass") + + @classmethod + def _puppet_concat(cls, it: PuppetConcat, indent: int) -> T: + raise NotImplementedError("puppet_concat must be implemented by subclass") + + @classmethod + def _puppet_collect(cls, it: PuppetCollect, indent: int) -> T: + raise NotImplementedError("puppet_collect must be implemented by subclass") + + @classmethod + def _puppet_if(cls, it: PuppetIf, indent: int) -> T: + raise NotImplementedError("puppet_if must be implemented by subclass") + + @classmethod + def _puppet_unless(cls, it: PuppetUnless, indent: int) -> T: + raise NotImplementedError("puppet_unless must be implemented by subclass") + + @classmethod + def _puppet_keyword(cls, it: PuppetKeyword, indent: int) -> T: + raise NotImplementedError("puppet_keyword must be implemented by subclass") + + @classmethod + def _puppet_exported_query(cls, it: PuppetExportedQuery, indent: int) -> T: + raise NotImplementedError("puppet_exported_query must be implemented by subclass") + + @classmethod + def _puppet_virtual_query(cls, it: PuppetVirtualQuery, indent: int) -> T: + raise NotImplementedError("puppet_virtual_query must be implemented by subclass") + + @classmethod + def _puppet_function(cls, it: PuppetFunction, indent: int) -> T: + raise NotImplementedError("puppet_function must be implemented by subclass") + + @classmethod + def _puppet_hash(cls, it: PuppetHash, indent: int) -> T: + raise NotImplementedError("puppet_hash must be implemented by subclass") + + @classmethod + def _puppet_heredoc(cls, it: PuppetHeredoc, indent: int) -> T: + raise NotImplementedError("puppet_heredoc must be implemented by subclass") + + @classmethod + def _puppet_literal_heredoc(cls, it: PuppetLiteralHeredoc, indent: int) -> T: + raise NotImplementedError("puppet_literal_heredoc must be implemented by subclass") + + @classmethod + def _puppet_var(cls, it: PuppetVar, indent: int) -> T: + raise NotImplementedError("puppet_var must be implemented by subclass") + + @classmethod + def _puppet_lambda(cls, it: PuppetLambda, indent: int) -> T: + raise NotImplementedError("puppet_lambda must be implemented by subclass") + + @classmethod + def _puppet_qn(cls, it: PuppetQn, indent: int) -> T: + raise NotImplementedError("puppet_qn must be implemented by subclass") + + @classmethod + def _puppet_qr(cls, it: PuppetQr, indent: int) -> T: + raise NotImplementedError("puppet_qr must be implemented by subclass") + + @classmethod + def _puppet_regex(cls, it: PuppetRegex, indent: int) -> T: + raise NotImplementedError("puppet_regex must be implemented by subclass") + + @classmethod + def _puppet_resource(cls, it: PuppetResource, indent: int) -> T: + raise NotImplementedError("puppet_resource must be implemented by subclass") + + @classmethod + def _puppet_define(cls, it: PuppetDefine, indent: int) -> T: + raise NotImplementedError("puppet_define must be implemented by subclass") + + @classmethod + def _puppet_string(cls, it: PuppetString, indent: int) -> T: + raise NotImplementedError("puppet_string must be implemented by subclass") + + @classmethod + def _puppet_number(cls, it: PuppetNumber, indent: int) -> T: + raise NotImplementedError("puppet_number must be implemented by subclass") + + @classmethod + def _puppet_invoke(cls, it: PuppetInvoke, indent: int) -> T: + raise NotImplementedError("puppet_invoke must be implemented by subclass") + + @classmethod + def _puppet_resource_defaults(cls, it: PuppetResourceDefaults, indent: int) -> T: + raise NotImplementedError("puppet_resource_defaults must be implemented by subclass") + + @classmethod + def _puppet_resource_override(cls, it: PuppetResourceOverride, indent: int) -> T: + raise NotImplementedError("puppet_resource_override must be implemented by subclass") + + @classmethod + def _puppet_declaration(cls, it: PuppetDeclaration, indent: int) -> T: + raise NotImplementedError("puppet_declaration must be implemented by subclass") + + @classmethod + def _puppet_selector(cls, it: PuppetSelector, indent: int) -> T: + raise NotImplementedError("puppet_selector must be implemented by subclass") + + @classmethod + def _puppet_block(cls, it: PuppetBlock, indent: int) -> T: + raise NotImplementedError("puppet_block must be implemented by subclass") + + @classmethod + def _puppet_node(cls, it: PuppetNode, indent: int) -> T: + raise NotImplementedError("puppet_node must be implemented by subclass") + + @classmethod + def _puppet_parenthesis(cls, it: PuppetParenthesis, indent: int) -> T: + raise NotImplementedError("puppet_parenthesis must be implemented by subclass") + + @classmethod + def _puppet_nop(cls, it: PuppetNop, indent: int) -> T: + raise NotImplementedError("puppet_nop must be implemented by subclass") + + @final + @classmethod + def serialize(cls, form: Puppet, indent: int) -> T: + """Dispatch depending on type.""" + match form: + case PuppetLiteral(): + return cls._puppet_literal(form, indent) + case PuppetAccess(): + return cls._puppet_access(form, indent) + case PuppetBinaryOperator(): + return cls._puppet_binary_operator(form, indent) + case PuppetUnaryOperator(): + return cls._puppet_unary_operator(form, indent) + case PuppetUnaryOperator(): + return cls._puppet_unary_operator(form, indent) + case PuppetArray(): + return cls._puppet_array(form, indent) + case PuppetCall(): + return cls._puppet_call(form, indent) + case PuppetCallMethod(): + return cls._puppet_call_method(form, indent) + case PuppetCase(): + return cls._puppet_case(form, indent) + case PuppetDeclarationParameter(): + return cls._puppet_declaration_parameter(form, indent) + case PuppetInstanciationParameter(): + return cls._puppet_instanciation_parameter(form, indent) + case PuppetClass(): + return cls._puppet_class(form, indent) + case PuppetConcat(): + return cls._puppet_concat(form, indent) + case PuppetCollect(): + return cls._puppet_collect(form, indent) + case PuppetIf(): + return cls._puppet_if(form, indent) + case PuppetUnless(): + return cls._puppet_unless(form, indent) + case PuppetKeyword(): + return cls._puppet_keyword(form, indent) + case PuppetExportedQuery(): + return cls._puppet_exported_query(form, indent) + case PuppetVirtualQuery(): + return cls._puppet_virtual_query(form, indent) + case PuppetFunction(): + return cls._puppet_function(form, indent) + case PuppetHash(): + return cls._puppet_hash(form, indent) + case PuppetHeredoc(): + return cls._puppet_heredoc(form, indent) + case PuppetLiteralHeredoc(): + return cls._puppet_literal_heredoc(form, indent) + case PuppetVar(): + return cls._puppet_var(form, indent) + case PuppetLambda(): + return cls._puppet_lambda(form, indent) + case PuppetQn(): + return cls._puppet_qn(form, indent) + case PuppetQr(): + return cls._puppet_qr(form, indent) + case PuppetRegex(): + return cls._puppet_regex(form, indent) + case PuppetResource(): + return cls._puppet_resource(form, indent) + case PuppetDefine(): + return cls._puppet_define(form, indent) + case PuppetString(): + return cls._puppet_string(form, indent) + case PuppetNumber(): + return cls._puppet_number(form, indent) + case PuppetInvoke(): + return cls._puppet_invoke(form, indent) + case PuppetResourceDefaults(): + return cls._puppet_resource_defaults(form, indent) + case PuppetResourceOverride(): + return cls._puppet_resource_override(form, indent) + case PuppetDeclaration(): + return cls._puppet_declaration(form, indent) + case PuppetSelector(): + return cls._puppet_selector(form, indent) + case PuppetBlock(): + return cls._puppet_block(form, indent) + case PuppetNode(): + return cls._puppet_node(form, indent) + case PuppetParenthesis(): + return cls._puppet_parenthesis(form, indent) + case PuppetNop(): + return cls._puppet_nop(form, indent) + case _: + logger.warn("Unexpected form: %s", form) + raise ValueError(f'Unexpected: {form}') diff --git a/muppet/puppet/format/html.py b/muppet/puppet/format/html.py new file mode 100644 index 0000000..9719b53 --- /dev/null +++ b/muppet/puppet/format/html.py @@ -0,0 +1,549 @@ +""" +Reserilaize AST as HTML. + +This is mostly an extension of the text formatter, but with some HTML +tags inserted. This is also why the text module is imported. + +.. code-block:: html + + <span class="{TYPE}">{BODY}</span> +""" + +import re +import logging +from .base import Serializer +from muppet.puppet.ast import ( + PuppetLiteral, PuppetAccess, PuppetBinaryOperator, + PuppetUnaryOperator, PuppetArray, PuppetCallMethod, + PuppetCase, PuppetDeclarationParameter, + PuppetInstanciationParameter, PuppetClass, PuppetConcat, + PuppetCollect, PuppetIf, PuppetUnless, PuppetKeyword, + PuppetExportedQuery, PuppetVirtualQuery, PuppetFunction, + PuppetHash, PuppetHeredoc, PuppetLiteralHeredoc, PuppetVar, + PuppetLambda, PuppetQn, PuppetQr, PuppetRegex, + PuppetResource, PuppetDefine, PuppetString, + PuppetNumber, PuppetInvoke, PuppetResourceDefaults, + PuppetResourceOverride, PuppetDeclaration, PuppetSelector, + PuppetBlock, PuppetNode, + PuppetCall, PuppetParenthesis, PuppetNop, + + HashEntry, + # PuppetParseError, +) +import html +from .text import ( + override, + find_heredoc_delimiter, + ind, + string_width, +) + + +logger = logging.getLogger(__name__) + + +def span(cls: str, content: str) -> str: + """Wrap content in a span, and escape content.""" + return f'<span class="{cls}">{html.escape(content)}</span>' + + +def literal(x: str) -> str: + """Tag string as a literal.""" + return span("literal", x) + + +def op(x: str) -> str: + """Tag string as an operator.""" + return span("op", x) + + +def keyword(x: str) -> str: + """Tag string as a keyword.""" + return span("keyword", x) + + +def var(x: str) -> str: + """Tag string as a variable.""" + return span("var", x) + + +def string(x: str) -> str: + """Tag strings as a string literal.""" + return span("string", x) + + +def number(x: str) -> str: + """Tag string as a number literal.""" + return span("number", x) + + +class HTMLFormatter(Serializer[str]): + """AST formatter returning source code.""" + + @classmethod + def format_declaration_parameter( + cls, + param: PuppetDeclarationParameter, + indent: int) -> str: + """Format a single declaration parameter.""" + out: str = '' + if param.type: + out += f'{cls.serialize(param.type, indent + 1)} ' + out += var(f'${param.k}') + if param.v: + out += f' = {cls.serialize(param.v, indent + 1)}' + return out + + @classmethod + def format_declaration_parameters( + cls, + lst: list[PuppetDeclarationParameter], + indent: int) -> str: + """ + Print declaration parameters. + + This formats the parameters for class, resoruce, and function declarations. + """ + if not lst: + return '' + + out = ' (\n' + for param in lst: + out += ind(indent + 1) + cls.format_declaration_parameter(param, indent + 1) + ',\n' + out += ind(indent) + ')' + return out + + @classmethod + def serialize_hash_entry( + cls, + entry: HashEntry, + indent: int) -> str: + """Return a hash entry as a string.""" + return f'{cls.serialize(entry.k, indent + 1)} => {cls.serialize(entry.v, indent + 2)}' + + @override + @classmethod + def _puppet_literal(cls, it: PuppetLiteral, indent: int) -> str: + return literal(it.literal) + + @override + @classmethod + def _puppet_access(cls, it: PuppetAccess, indent: int) -> str: + args = ', '.join(cls.serialize(x, indent) for x in it.args) + + return f'{cls.serialize(it.how, indent)}[{args}]' + + @override + @classmethod + def _puppet_binary_operator(cls, it: PuppetBinaryOperator, indent: int) -> str: + out = cls.serialize(it.lhs, indent) + out += f' {op(it.op)} ' + out += cls.serialize(it.rhs, indent) + return out + + @override + @classmethod + def _puppet_unary_operator(cls, it: PuppetUnaryOperator, indent: int) -> str: + return f'{op(it.op)} {cls.serialize(it.x, indent)}' + + @override + @classmethod + def _puppet_array(cls, it: PuppetArray, indent: int) -> str: + if not it.items: + return '[]' + else: + out = '[\n' + for item in it.items: + out += ind(indent + 1) + cls.serialize(item, indent + 2) + ',\n' + out += ind(indent) + ']' + return out + + @override + @classmethod + def _puppet_call(cls, it: PuppetCall, indent: int) -> str: + args = ', '.join(cls.serialize(x, indent) for x in it.args) + return f'{cls.serialize(it.func, indent)}({args})' + + @override + @classmethod + def _puppet_call_method(cls, it: PuppetCallMethod, indent: int) -> str: + out: str = cls.serialize(it.func, indent) + + if it.args: + args = ', '.join(cls.serialize(x, indent) for x in it.args) + out += f' ({args})' + + if it.block: + out += cls.serialize(it.block, indent) + + return out + + @override + @classmethod + def _puppet_case(cls, it: PuppetCase, indent: int) -> str: + out: str = f'{keyword("case")} {cls.serialize(it.test, indent)} {{\n' + for (when, body) in it.cases: + out += ind(indent + 1) + out += ', '.join(cls.serialize(x, indent + 1) for x in when) + out += ': {\n' + for item in body: + out += ind(indent + 2) + cls.serialize(item, indent + 2) + '\n' + out += ind(indent + 1) + '}\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_declaration_parameter(cls, it: PuppetDeclarationParameter, indent: int) -> str: + out: str = '' + if it.type: + out += f'{cls.serialize(it.type, indent + 1)} ' + out += var(f'${it.k}') + if it.v: + out += f' = {cls.serialize(it.v, indent + 1)}' + return out + + @override + @classmethod + def _puppet_instanciation_parameter(cls, it: PuppetInstanciationParameter, indent: int) -> str: + return f'{it.k} {it.arrow} {cls.serialize(it.v, indent)}' + + @override + @classmethod + def _puppet_class(cls, it: PuppetClass, indent: int) -> str: + out: str = f'{keyword("class")} {it.name}' + if it.params: + out += cls.format_declaration_parameters(it.params, indent) + + out += ' {\n' + for form in it.body: + out += ind(indent+1) + cls.serialize(form, indent+1) + '\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_concat(cls, it: PuppetConcat, indent: int) -> str: + out = '"' + for item in it.fragments: + match item: + case PuppetString(s): + out += s + case PuppetVar(x): + out += var(f"${{{x}}}") + case puppet: + out += f"${{{cls.serialize(puppet, indent)}}}" + out += '"' + # Don't escape `out`, since it contains sub-expressions + return f'<span class="string">{out}</span>' + + @override + @classmethod + def _puppet_collect(cls, it: PuppetCollect, indent: int) -> str: + return f'{cls.serialize(it.type, indent)} {cls.serialize(it.query, indent + 1)}' + + @override + @classmethod + def _puppet_if(cls, it: PuppetIf, indent: int) -> str: + out: str = f'{keyword("if")} {cls.serialize(it.condition, indent)} {{\n' + for item in it.consequent: + out += ind(indent+1) + cls.serialize(item, indent+1) + '\n' + out += ind(indent) + '}' + if alts := it.alternative: + # TODO elsif + out += f' {keyword("else")} {{\n' + for item in alts: + out += ind(indent+1) + cls.serialize(item, indent+1) + '\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_unless(cls, it: PuppetUnless, indent: int) -> str: + out: str = f'{keyword("unless")} {cls.serialize(it.condition, indent)} {{\n' + for item in it.consequent: + out += ind(indent+1) + cls.serialize(item, indent+1) + '\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_keyword(cls, it: PuppetKeyword, indent: int) -> str: + return it.name + + @override + @classmethod + def _puppet_exported_query(cls, it: PuppetExportedQuery, indent: int) -> str: + out: str = op('<<|') + if f := it.filter: + out += ' ' + cls.serialize(f, indent) + out += ' ' + op('|>>') + return out + + @override + @classmethod + def _puppet_virtual_query(cls, it: PuppetVirtualQuery, indent: int) -> str: + out: str = op('<|') + if f := it.q: + out += ' ' + cls.serialize(f, indent) + out += ' ' + op('|>') + return out + + @override + @classmethod + def _puppet_function(cls, it: PuppetFunction, indent: int) -> str: + out: str = f'{keyword("function")} {it.name}' + if it.params: + out += cls.format_declaration_parameters(it.params, indent) + + if ret := it.returns: + out += f' {op(">>")} {cls.serialize(ret, indent + 1)}' + + out += ' {\n' + for item in it.body: + out += ind(indent + 1) + cls.serialize(item, indent + 1) + '\n' + out += ind(indent) + '}' + + return out + + @override + @classmethod + def _puppet_hash(cls, it: PuppetHash, indent: int) -> str: + if not it.entries: + return '{}' + else: + out: str = '{\n' + for item in it.entries: + out += ind(indent + 1) + out += cls.serialize_hash_entry(item, indent + 1) + out += ',\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_heredoc(cls, it: PuppetHeredoc, indent: int) -> str: + """ + Serialize heredoc with interpolation. + + The esacpes $, r, and t are always added and un-escaped, + while the rest are left as is, since they work fine in the literal. + """ + syntax: str = '' + if it.syntax: + syntax = f':{it.syntax}' + + # TODO find delimiter + body = '' + for frag in it.fragments: + match frag: + case PuppetString(s): + # \r, \t, \, $ + e = re.sub('[\r\t\\\\$]', lambda m: { + '\r': r'\r', + '\t': r'\t', + }.get(m[0], '\\' + m[0]), s) + body += e + case PuppetVar(x): + body += f'${{{x}}}' + case p: + body += cls.serialize(p, indent + 2) + + # Check if string ends with a newline + match it.fragments[-1]: + case PuppetString(s) if s.endswith('\n'): + eol_marker = '' + body = body[:-1] + case _: + eol_marker = '-' + + # Aligning this to the left column is ugly, but saves us from + # parsing newlines in the actual string + return f'@("EOF"{syntax}/$rt)\n{body}\n|{eol_marker} EOF' + + @override + @classmethod + def _puppet_literal_heredoc(cls, it: PuppetLiteralHeredoc, indent: int) -> str: + syntax: str = '' + if it.syntax: + syntax = f':{it.syntax}' + + out: str = '' + if not it.content: + out += f'@(EOF{syntax})\n' + out += ind(indent) + '|- EOF' + return out + + delimiter = find_heredoc_delimiter(it.content) + + out += f'@({delimiter}{syntax})\n' + + lines = it.content.split('\n') + eol: bool = False + if lines[-1] == '': + lines = lines[:-1] # Remove last + eol = True + + for line in lines: + out += ind(indent + 1) + line + '\n' + + out += ind(indent + 1) + '|' + + if not eol: + out += '-' + + out += ' ' + delimiter + + return out + + @override + @classmethod + def _puppet_var(cls, it: PuppetVar, indent: int) -> str: + return var(f'${it.name}') + + @override + @classmethod + def _puppet_lambda(cls, it: PuppetLambda, indent: int) -> str: + out: str = '|' + for item in it.params: + out += 'TODO' + out += '| {' + for form in it.body: + out += ind(indent + 1) + cls.serialize(form, indent + 1) + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_qn(cls, it: PuppetQn, indent: int) -> str: + return span('qn', it.name) + + @override + @classmethod + def _puppet_qr(cls, it: PuppetQr, indent: int) -> str: + return span('qn', it.name) + + @override + @classmethod + def _puppet_regex(cls, it: PuppetRegex, indent: int) -> str: + return span('regex', f'/{it.s}/') + + @override + @classmethod + def _puppet_resource(cls, it: PuppetResource, indent: int) -> str: + out = f'{cls.serialize(it.type, indent + 1)} {{' + match it.bodies: + case [(name, values)]: + out += f' {cls.serialize(name, indent + 1)}:\n' + for v in values: + out += ind(indent + 1) + cls.serialize(v, indent + 2) + ',\n' + case bodies: + out += '\n' + for (name, values) in bodies: + out += f'{ind(indent + 1)}{cls.serialize(name, indent + 1)}:\n' + for v in values: + out += ind(indent + 2) + cls.serialize(v, indent + 3) + ',\n' + out += ind(indent + 2) + ';\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_define(cls, it: PuppetDefine, indent: int) -> str: + out: str = f'{keyword("define")} {it.name}' + if params := it.params: + out += cls.format_declaration_parameters(params, indent) + + out += ' {\n' + for form in it.body: + out += ind(indent + 1) + cls.serialize(form, indent + 1) + '\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_string(cls, it: PuppetString, indent: int) -> str: + # TODO escaping + return string(f"'{it.s}'") + + @override + @classmethod + def _puppet_number(cls, it: PuppetNumber, indent: int) -> str: + return number(str(it.x)) + + @override + @classmethod + def _puppet_invoke(cls, it: PuppetInvoke, indent: int) -> str: + invoker = f'{cls.serialize(it.func, indent)}' + out: str = invoker + template: str + if invoker == keyword('include'): + template = ' {}' + else: + template = '({})' + out += template.format(', '.join(cls.serialize(x, indent + 1) for x in it.args)) + return out + + @override + @classmethod + def _puppet_resource_defaults(cls, it: PuppetResourceDefaults, indent: int) -> str: + out: str = f'{cls.serialize(it.type, indent)} {{\n' + for op in it.ops: + out += ind(indent + 1) + cls.serialize(op, indent + 1) + ',\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_resource_override(cls, it: PuppetResourceOverride, indent: int) -> str: + out: str = f'{cls.serialize(it.resource, indent)} {{\n' + for op in it.ops: + out += ind(indent + 1) + cls.serialize(op, indent + 1) + ',\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_declaration(cls, it: PuppetDeclaration, indent: int) -> str: + return f'{cls.serialize(it.k, indent)} = {cls.serialize(it.v, indent)}' + + @override + @classmethod + def _puppet_selector(cls, it: PuppetSelector, indent: int) -> str: + out: str = f'{cls.serialize(it.resource, indent)} ? {{\n' + rendered_cases = [(cls.serialize(test, indent + 1), + cls.serialize(body, indent + 2)) + for (test, body) in it.cases] + case_width = max(string_width(c[0], indent + 1) for c in rendered_cases) + for (test, body) in rendered_cases: + out += ind(indent + 1) + test + out += ' ' * (case_width - string_width(test, indent + 1)) + out += f' => {body},\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_block(cls, it: PuppetBlock, indent: int) -> str: + return '\n'.join(cls.serialize(x, indent) for x in it.entries) + + @override + @classmethod + def _puppet_node(cls, it: PuppetNode, indent: int) -> str: + out: str = keyword('node') + ' ' + out += ', '.join(cls.serialize(x, indent) for x in it.matches) + out += ' {\n' + for item in it.body: + out += ind(indent + 1) + cls.serialize(item, indent + 1) + '\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_parenthesis(cls, it: PuppetParenthesis, indent: int) -> str: + return f'({cls.serialize(it.form, indent)})' + + @override + @classmethod + def _puppet_nop(cls, it: PuppetNop, indent: int) -> str: + return '' diff --git a/muppet/puppet/format/text.py b/muppet/puppet/format/text.py new file mode 100644 index 0000000..a3e772d --- /dev/null +++ b/muppet/puppet/format/text.py @@ -0,0 +1,594 @@ +""" +AST serializer returning source code. + +Can be used as a "bad" pretty printer (bad, since comments are discarded). +""" + +import re +import uuid +import logging +from .base import Serializer +from muppet.puppet.ast import ( + PuppetLiteral, PuppetAccess, PuppetBinaryOperator, + PuppetUnaryOperator, PuppetArray, PuppetCallMethod, + PuppetCase, PuppetDeclarationParameter, + PuppetInstanciationParameter, PuppetClass, PuppetConcat, + PuppetCollect, PuppetIf, PuppetUnless, PuppetKeyword, + PuppetExportedQuery, PuppetVirtualQuery, PuppetFunction, + PuppetHash, PuppetHeredoc, PuppetLiteralHeredoc, PuppetVar, + PuppetLambda, PuppetQn, PuppetQr, PuppetRegex, + PuppetResource, PuppetDefine, PuppetString, + PuppetNumber, PuppetInvoke, PuppetResourceDefaults, + PuppetResourceOverride, PuppetDeclaration, PuppetSelector, + PuppetBlock, PuppetNode, + PuppetCall, PuppetParenthesis, PuppetNop, + + HashEntry, + # PuppetParseError, +) + +from typing import ( + TypeVar, + Callable, +) + + +F = TypeVar('F', bound=Callable[..., object]) + +# TODO replace this decorator with +# from typing import override +# once the target python version is changed to 3.12 + + +def override(f: F) -> F: + """ + Return function unchanged. + + Placeholder @override annotator if the actual annotation isn't + implemented in the current python version. + """ + return f + + +logger = logging.getLogger(__name__) + + +def find_heredoc_delimiter( + source: str, + options: list[str] = ['EOF', 'EOL', 'STR', 'END'], + ) -> str: + """ + Find a suitable heredoc delimiter for the given string. + + Heredoc's are delimited like + + .. code-block:: puppet + + @(EOF) + Some text + here + | EOF + + This looks through the text, and finds a suitable marker (``EOF`` + here) which isn't present in the text. It first tries each given + option, and then randomizes until it finds one. + + :param source: + String to search for collisions. + :param options: + Prefered delimiters, with descending priority. + :returns: + A string like EOF, guaranteed to not be present in the source. + """ + for option in options: + if option not in source: + return option + + while True: + delim = uuid.uuid4().hex + if delim not in source: + return delim + + +def ind(level: int) -> str: + """Return indentation string of given depth.""" + return ' ' * level * 2 + + +def string_width(s: str, indent: int) -> int: + """ + Return the width of a rendered puppet expression. + + In a perfect world, this would return the rendered width of an + expression, as in the total amount of columns between its leftmost + and rightmost (printed) character. For example, the case below + should return 4, and any extra highlight to the left would be + discarded according to ``indent``. + + .. todo:: + + The smart width "algorithm" is currently not implemented, + instead, the length of the string is returned. + + .. code-block:: puppet + + [ + 1, + 2, + ] + + + :param s: + The rendered puppet expression + :param indent: + The indentation level which was used when creating the string. + """ + return len(s) + + +class TextFormatter(Serializer[str]): + """AST formatter returning source code.""" + + @classmethod + def format_declaration_parameter( + cls, + param: PuppetDeclarationParameter, + indent: int) -> str: + """Format a single declaration parameter.""" + out: str = '' + if param.type: + out += f'{cls.serialize(param.type, indent + 1)} ' + out += f'${param.k}' + if param.v: + out += f' = {cls.serialize(param.v, indent + 1)}' + return out + + @classmethod + def format_declaration_parameters( + cls, + lst: list[PuppetDeclarationParameter], + indent: int) -> str: + """ + Print declaration parameters. + + This formats the parameters for class, resoruce, and function declarations. + """ + if not lst: + return '' + + out = ' (\n' + for param in lst: + out += ind(indent + 1) + cls.format_declaration_parameter(param, indent + 1) + ',\n' + out += ind(indent) + ')' + return out + + @classmethod + def serialize_hash_entry( + cls, + entry: HashEntry, + indent: int) -> str: + """Return a hash entry as a string.""" + return f'{cls.serialize(entry.k, indent + 1)} => {cls.serialize(entry.v, indent + 2)}' + + @override + @classmethod + def _puppet_literal(cls, it: PuppetLiteral, indent: int) -> str: + return it.literal + + @override + @classmethod + def _puppet_access(cls, it: PuppetAccess, indent: int) -> str: + args = ', '.join(cls.serialize(x, indent) for x in it.args) + + return f'{cls.serialize(it.how, indent)}[{args}]' + + @override + @classmethod + def _puppet_binary_operator(cls, it: PuppetBinaryOperator, indent: int) -> str: + return f'{cls.serialize(it.lhs, indent)} {it.op} {cls.serialize(it.rhs, indent)}' + + @override + @classmethod + def _puppet_unary_operator(cls, it: PuppetUnaryOperator, indent: int) -> str: + return f'{it.op} {cls.serialize(it.x, indent)}' + + @override + @classmethod + def _puppet_array(cls, it: PuppetArray, indent: int) -> str: + if not it.items: + return '[]' + else: + out = '[\n' + for item in it.items: + out += ind(indent + 1) + cls.serialize(item, indent + 2) + ',\n' + out += ind(indent) + ']' + return out + + @override + @classmethod + def _puppet_call(cls, it: PuppetCall, indent: int) -> str: + args = ', '.join(cls.serialize(x, indent) for x in it.args) + return f'{cls.serialize(it.func, indent)}({args})' + + @override + @classmethod + def _puppet_call_method(cls, it: PuppetCallMethod, indent: int) -> str: + out: str = cls.serialize(it.func, indent) + + if it.args: + args = ', '.join(cls.serialize(x, indent) for x in it.args) + out += f' ({args})' + + if it.block: + out += cls.serialize(it.block, indent) + + return out + + @override + @classmethod + def _puppet_case(cls, it: PuppetCase, indent: int) -> str: + out: str = f'case {cls.serialize(it.test, indent)} {{\n' + for (when, body) in it.cases: + out += ind(indent + 1) + out += ', '.join(cls.serialize(x, indent + 1) for x in when) + out += ': {\n' + for item in body: + out += ind(indent + 2) + cls.serialize(item, indent + 2) + '\n' + out += ind(indent + 1) + '}\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_declaration_parameter(cls, it: PuppetDeclarationParameter, indent: int) -> str: + out: str = '' + if it.type: + out += f'{cls.serialize(it.type, indent + 1)} ' + out += f'${it.k}' + if it.v: + out += f' = {cls.serialize(it.v, indent + 1)}' + return out + + @override + @classmethod + def _puppet_instanciation_parameter(cls, it: PuppetInstanciationParameter, indent: int) -> str: + return f'{it.k} {it.arrow} {cls.serialize(it.v, indent)}' + + @override + @classmethod + def _puppet_class(cls, it: PuppetClass, indent: int) -> str: + out: str = f'class {it.name}' + if it.params: + out += cls.format_declaration_parameters(it.params, indent) + + out += ' {\n' + for form in it.body: + out += ind(indent+1) + cls.serialize(form, indent+1) + '\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_concat(cls, it: PuppetConcat, indent: int) -> str: + out = '"' + for item in it.fragments: + match item: + case PuppetString(s): + out += s + case PuppetVar(x): + out += f"${{{x}}}" + case puppet: + out += f"${{{cls.serialize(puppet, indent)}}}" + out += '"' + return out + + @override + @classmethod + def _puppet_collect(cls, it: PuppetCollect, indent: int) -> str: + return f'{cls.serialize(it.type, indent)} {cls.serialize(it.query, indent + 1)}' + + @override + @classmethod + def _puppet_if(cls, it: PuppetIf, indent: int) -> str: + out: str = f'if {cls.serialize(it.condition, indent)} {{\n' + for item in it.consequent: + out += ind(indent+1) + cls.serialize(item, indent+1) + '\n' + out += ind(indent) + '}' + if alts := it.alternative: + # TODO elsif + out += ' else {\n' + for item in alts: + out += ind(indent+1) + cls.serialize(item, indent+1) + '\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_unless(cls, it: PuppetUnless, indent: int) -> str: + out: str = f'unless {cls.serialize(it.condition, indent)} {{\n' + for item in it.consequent: + out += ind(indent+1) + cls.serialize(item, indent+1) + '\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_keyword(cls, it: PuppetKeyword, indent: int) -> str: + return it.name + + @override + @classmethod + def _puppet_exported_query(cls, it: PuppetExportedQuery, indent: int) -> str: + out: str = '<<|' + if f := it.filter: + out += ' ' + cls.serialize(f, indent) + out += ' |>>' + return out + + @override + @classmethod + def _puppet_virtual_query(cls, it: PuppetVirtualQuery, indent: int) -> str: + out: str = '<|' + if f := it.q: + out += ' ' + cls.serialize(f, indent) + out += ' |>' + return out + + @override + @classmethod + def _puppet_function(cls, it: PuppetFunction, indent: int) -> str: + out: str = f'function {it.name}' + if it.params: + out += cls.format_declaration_parameters(it.params, indent) + + if ret := it.returns: + out += f' >> {cls.serialize(ret, indent + 1)}' + + out += ' {\n' + for item in it.body: + out += ind(indent + 1) + cls.serialize(item, indent + 1) + '\n' + out += ind(indent) + '}' + + return out + + @override + @classmethod + def _puppet_hash(cls, it: PuppetHash, indent: int) -> str: + if not it.entries: + return '{}' + else: + out: str = '{\n' + for item in it.entries: + out += ind(indent + 1) + out += cls.serialize_hash_entry(item, indent + 1) + out += ',\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_heredoc(cls, it: PuppetHeredoc, indent: int) -> str: + """ + Serialize heredoc with interpolation. + + The esacpes $, r, and t are always added and un-escaped, + while the rest are left as is, since they work fine in the literal. + """ + syntax: str = '' + if it.syntax: + syntax = f':{it.syntax}' + + # TODO find delimiter + body = '' + for frag in it.fragments: + match frag: + case PuppetString(s): + # \r, \t, \, $ + e = re.sub('[\r\t\\\\$]', lambda m: { + '\r': r'\r', + '\t': r'\t', + }.get(m[0], '\\' + m[0]), s) + body += e + case PuppetVar(x): + body += f'${{{x}}}' + case p: + body += cls.serialize(p, indent + 2) + + # Check if string ends with a newline + match it.fragments[-1]: + case PuppetString(s) if s.endswith('\n'): + eol_marker = '' + body = body[:-1] + case _: + eol_marker = '-' + + # Aligning this to the left column is ugly, but saves us from + # parsing newlines in the actual string + return f'@("EOF"{syntax}/$rt)\n{body}\n|{eol_marker} EOF' + + @override + @classmethod + def _puppet_literal_heredoc(cls, it: PuppetLiteralHeredoc, indent: int) -> str: + syntax: str = '' + if it.syntax: + syntax = f':{it.syntax}' + + out: str = '' + if not it.content: + out += f'@(EOF{syntax})\n' + out += ind(indent) + '|- EOF' + return out + + delimiter = find_heredoc_delimiter(it.content) + + out += f'@({delimiter}{syntax})\n' + + lines = it.content.split('\n') + eol: bool = False + if lines[-1] == '': + lines = lines[:-1] # Remove last + eol = True + + for line in lines: + out += ind(indent + 1) + line + '\n' + + out += ind(indent + 1) + '|' + + if not eol: + out += '-' + + out += ' ' + delimiter + + return out + + @override + @classmethod + def _puppet_var(cls, it: PuppetVar, indent: int) -> str: + return f'${it.name}' + + @override + @classmethod + def _puppet_lambda(cls, it: PuppetLambda, indent: int) -> str: + out: str = '|' + for item in it.params: + out += 'TODO' + out += '| {' + for form in it.body: + out += ind(indent + 1) + cls.serialize(form, indent + 1) + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_qn(cls, it: PuppetQn, indent: int) -> str: + return it.name + + @override + @classmethod + def _puppet_qr(cls, it: PuppetQr, indent: int) -> str: + return it.name + + @override + @classmethod + def _puppet_regex(cls, it: PuppetRegex, indent: int) -> str: + return f'/{it.s}/' + + @override + @classmethod + def _puppet_resource(cls, it: PuppetResource, indent: int) -> str: + out = f'{cls.serialize(it.type, indent + 1)} {{' + match it.bodies: + case [(name, values)]: + out += f' {cls.serialize(name, indent + 1)}:\n' + for v in values: + out += ind(indent + 1) + cls.serialize(v, indent + 2) + ',\n' + case bodies: + out += '\n' + for (name, values) in bodies: + out += f'{ind(indent + 1)}{cls.serialize(name, indent + 1)}:\n' + for v in values: + out += ind(indent + 2) + cls.serialize(v, indent + 3) + ',\n' + out += ind(indent + 2) + ';\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_define(cls, it: PuppetDefine, indent: int) -> str: + out: str = f'define {it.name}' + if params := it.params: + out += cls.format_declaration_parameters(params, indent) + + out += ' {\n' + for form in it.body: + out += ind(indent + 1) + cls.serialize(form, indent + 1) + '\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_string(cls, it: PuppetString, indent: int) -> str: + # TODO escaping + return f"'{it.s}'" + + @override + @classmethod + def _puppet_number(cls, it: PuppetNumber, indent: int) -> str: + return str(it.x) + + @override + @classmethod + def _puppet_invoke(cls, it: PuppetInvoke, indent: int) -> str: + invoker = f'{cls.serialize(it.func, indent)}' + out: str = invoker + template: str + if invoker == 'include': + template = ' {}' + else: + template = '({})' + out += template.format(', '.join(cls.serialize(x, indent + 1) for x in it.args)) + return out + + @override + @classmethod + def _puppet_resource_defaults(cls, it: PuppetResourceDefaults, indent: int) -> str: + out: str = f'{cls.serialize(it.type, indent)} {{\n' + for op in it.ops: + out += ind(indent + 1) + cls.serialize(op, indent + 1) + ',\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_resource_override(cls, it: PuppetResourceOverride, indent: int) -> str: + out: str = f'{cls.serialize(it.resource, indent)} {{\n' + for op in it.ops: + out += ind(indent + 1) + cls.serialize(op, indent + 1) + ',\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_declaration(cls, it: PuppetDeclaration, indent: int) -> str: + return f'{cls.serialize(it.k, indent)} = {cls.serialize(it.v, indent)}' + + @override + @classmethod + def _puppet_selector(cls, it: PuppetSelector, indent: int) -> str: + out: str = f'{cls.serialize(it.resource, indent)} ? {{\n' + rendered_cases = [(cls.serialize(test, indent + 1), + cls.serialize(body, indent + 2)) + for (test, body) in it.cases] + case_width = max(string_width(c[0], indent + 1) for c in rendered_cases) + for (test, body) in rendered_cases: + out += ind(indent + 1) + test + out += ' ' * (case_width - string_width(test, indent + 1)) + out += f' => {body},\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_block(cls, it: PuppetBlock, indent: int) -> str: + return '\n'.join(cls.serialize(x, indent) for x in it.entries) + + @override + @classmethod + def _puppet_node(cls, it: PuppetNode, indent: int) -> str: + out: str = 'node ' + out += ', '.join(cls.serialize(x, indent) for x in it.matches) + out += ' {\n' + for item in it.body: + out += ind(indent + 1) + cls.serialize(item, indent + 1) + '\n' + out += ind(indent) + '}' + return out + + @override + @classmethod + def _puppet_parenthesis(cls, it: PuppetParenthesis, indent: int) -> str: + return f'({cls.serialize(it.form, indent)})' + + @override + @classmethod + def _puppet_nop(cls, it: PuppetNop, indent: int) -> str: + return '' diff --git a/muppet/puppet/parser.py b/muppet/puppet/parser.py index b3724eb..fb8d14e 100644 --- a/muppet/puppet/parser.py +++ b/muppet/puppet/parser.py @@ -12,6 +12,7 @@ import json from typing import Any, TypeAlias, Union from ..cache import Cache import logging +from collections import OrderedDict logger = logging.getLogger(__name__) @@ -21,7 +22,7 @@ logger = logging.getLogger(__name__) cache = Cache('/home/hugo/.cache/muppet-strings') -def tagged_list_to_dict(lst: list[Any]) -> dict[Any, Any]: +def tagged_list_to_dict(lst: list[Any]) -> OrderedDict[Any, Any]: """ Turn a tagged list into a dictionary. @@ -33,8 +34,8 @@ def tagged_list_to_dict(lst: list[Any]) -> dict[Any, Any]: >>> tagged_list_to_dict(['a', 1, 'b', 2]) {'a': 1, 'b': 2} """ - return {lst[i]: lst[i+1] - for i in range(0, len(lst), 2)} + return OrderedDict((lst[i], lst[i+1]) + for i in range(0, len(lst), 2)) def traverse(tree: Any) -> Any: @@ -57,9 +58,9 @@ def traverse(tree: Any) -> Any: # `x in tree` pattern since there may be empty lists (which # are "False") if '#' in tree: - return {key: traverse(value) - for (key, value) - in tagged_list_to_dict(tree['#']).items()} + return OrderedDict((key, traverse(value)) + for (key, value) + in tagged_list_to_dict(tree['#']).items()) elif '^' in tree: return [traverse(subtree) for subtree in tree['^']] else: @@ -109,23 +110,3 @@ def puppet_parser(code: str) -> JSON: return data else: raise ValueError('Expected well formed tree, got %s', data) - - -def __main() -> None: - import sys - match sys.argv: - case [_]: - inp = sys.stdin - case [_, file]: - inp = open(file) - case _: - raise Exception("This is impossible to rearch") - - json.dump(puppet_parser(inp.read()), - sys.stdout, - indent=2) - print() - - -if __name__ == '__main__': - __main() diff --git a/tests/test_ast.py b/tests/test_ast.py new file mode 100644 index 0000000..593e0f6 --- /dev/null +++ b/tests/test_ast.py @@ -0,0 +1,633 @@ +""" +Test for building, and reserializing our Puppet ASTs. + +All Puppet "child" declared in :py:mod:`muppet.puppet.ast` +should be tested here, with the exceptions noted below. + +Note that PuppetParseError isn't tested, since there should be no way +to create those objects through ``build_ast``. + +Skipped +------- + +The following forms lack a dedicated test, for the reasons +stated bellow. + +exported query & virtual query + Only makes since in a collect statement + +lambda + Can only exist in certain contexts, so they are covered there. + +qn, qr, nop + Only exists within other forms. + +str + Not a true form, but part of string interpolation forms. +""" + +from muppet.puppet.ast import ( + Puppet, build_ast, + + PuppetLiteral, PuppetAccess, PuppetBinaryOperator, + PuppetUnaryOperator, PuppetArray, PuppetCallMethod, + PuppetCase, PuppetDeclarationParameter, + PuppetInstanciationParameter, PuppetClass, PuppetConcat, + PuppetCollect, PuppetIf, PuppetUnless, PuppetKeyword, + PuppetExportedQuery, PuppetVirtualQuery, PuppetFunction, + PuppetHash, PuppetHeredoc, PuppetLiteralHeredoc, PuppetVar, + PuppetLambda, PuppetQn, PuppetQr, PuppetRegex, + PuppetResource, PuppetDefine, PuppetString, + PuppetNumber, PuppetInvoke, PuppetResourceDefaults, + PuppetResourceOverride, PuppetDeclaration, PuppetSelector, + PuppetBlock, PuppetNode, PuppetCall, + + HashEntry, + + # # These should be tested, if I figure out how to cause them + # , PuppetParenthesis, + # , PuppetNop + + # # This is intentionally ignored, since it should be impossible + # # to cause. + # , PuppetParseError +) +import muppet.puppet.parser +from muppet.puppet.format import serialize +from muppet.puppet.format.text import TextFormatter +import pytest + + +def parse(puppet_source: str) -> Puppet: + """Shorthand for running the parser in tests.""" + return build_ast(muppet.puppet.parser.puppet_parser( + puppet_source)) + +def ser(ast: Puppet) -> str: + return serialize(ast, TextFormatter) + + +# from pprint import pprint +# def run(x): +# """Function for generating test cases.""" +# pprint(parse(x)) + + +def test_literal(): + s1 = "true" + r1 = PuppetLiteral('true') + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_keyword(): + s1 = "default" + r1 = PuppetKeyword("default") + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_var(): + s1 = "$x" + r1 = PuppetVar(name='x') + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_string(): + s1 = "'Hello'" + r1 = PuppetString(s='Hello') + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_concat(): + s1 = '"Hello ${name}"' + s2 = '"Hello ${$x + 1}"' + s3 = '"Hello ${4}"' + + r1 = PuppetConcat(fragments=[PuppetString(s='Hello '), + PuppetVar(name='name')]) + r2 = PuppetConcat(fragments=[PuppetString(s='Hello '), + PuppetBinaryOperator(op='+', + lhs=PuppetVar(name='x'), + rhs=PuppetNumber(x=1))]) + r3 = PuppetConcat(fragments=[PuppetString(s='Hello '), PuppetVar(name='4')]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + assert parse(s2) == r2 + assert ser(r2) == s2 + assert parse(s3) == r3 + assert ser(r3) == s3 + + +def test_access(): + s1 = "$x[1]" + r1 = PuppetAccess(how=PuppetVar(name='x'), args=[PuppetNumber(x=1)]) + + s2 = "$x[1, 2]" + r2 = PuppetAccess(how=PuppetVar(name='x'), + args=[PuppetNumber(x=1), PuppetNumber(x=2)]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + assert parse(s2) == r2 + assert ser(r2) == s2 + + +def test_bin_op(): + s1 = "$x = $y" + r1 = PuppetDeclaration(k=PuppetVar(name='x'), v=PuppetVar(name='y')) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_un_op(): + s1 = "- $x" + r1 = PuppetUnaryOperator(op='-', x=PuppetVar(name='x')) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_array(): + s1 = '[]' + r1 = PuppetArray([]) + + s2 = """ +[ + 1, + 2, +] + """.strip() + r2 = PuppetArray(items=[PuppetNumber(x=1), + PuppetNumber(x=2)]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + assert parse(s2) == r2 + assert ser(r2) == s2 + + +def test_call(): + s1 = "$x = f(1, 5)" + r1 = PuppetDeclaration(PuppetVar('x'), + PuppetCall(PuppetQn('f'), + [PuppetNumber(1), + PuppetNumber(5)])) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +@pytest.mark.xfail(reason=". is treated as a binary operator, so spaces are added") +def test_call_method(): + """ + Test method calls. + + This method also covers lambda. + """ + s1 = 'm.f()' + s2 = 'm.f(1)' + s3 = 'm.f |$x| { 1 }' + + r1 = PuppetCallMethod(func=PuppetBinaryOperator(op='.', + lhs=PuppetQn(name='m'), + rhs=PuppetQn(name='f')), + args=[]) + r2 = PuppetCallMethod(func=PuppetBinaryOperator(op='.', + lhs=PuppetQn(name='m'), + rhs=PuppetQn(name='f')), + args=[PuppetNumber(x=1)]) + r3 = PuppetCallMethod(func=PuppetBinaryOperator(op='.', + lhs=PuppetQn(name='m'), + rhs=PuppetQn(name='f')), + args=[], + block=PuppetLambda(params=[PuppetDeclarationParameter(k='x')], + body=[PuppetNumber(x=1)])) + + assert parse(s1) == r1 + assert ser(r1) == s1 + assert parse(s2) == r2 + assert ser(r2) == s2 + + assert parse(s3) == r3 + assert ser(r3) == s3 + + +def test_case(): + s1 = """ +case 1 { + 'a': { + 1 + } + /b/, /c/: { + 2 + } +} + """.strip() + + r1 = PuppetCase(test=PuppetNumber(x=1), + cases=[([PuppetString(s='a')], [PuppetNumber(x=1)]), + ([PuppetRegex(s='b'), PuppetRegex(s='c')], + [PuppetNumber(x=2)])]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_collect(): + s1 = "File <| 10 |>" + s2 = "File <<| 20 |>>" + + r1 = PuppetCollect(type=PuppetQr(name='File'), + query=PuppetVirtualQuery(q=PuppetNumber(x=10))) + r2 = PuppetCollect(type=PuppetQr(name='File'), + query=PuppetExportedQuery(filter=PuppetNumber(x=20))) + + assert parse(s1) == r1 + assert ser(r1) == s1 + assert parse(s2) == r2 + assert ser(r2) == s2 + + +def test_if(): + s1 = """ +if 1 { + 'a' +} + """.strip() + + s2 = """ +if 1 { + 'a' +} else { + 'b' +} + """.strip() + + s3 = """ +if 1 { + 'a' +} elsif 2 { + 'b' +} else { + 'c' +} + """.strip() + + r1 = PuppetIf(condition=PuppetNumber(x=1), + consequent=[PuppetString(s='a')]) + + r2 = PuppetIf(condition=PuppetNumber(x=1), + consequent=[PuppetString(s='a')], + alternative=[PuppetString(s='b')]) + + r3 = PuppetIf(condition=PuppetNumber(x=1), + consequent=[PuppetString(s='a')], + alternative=[PuppetIf(condition=PuppetNumber(x=2), + consequent=[PuppetString(s='b')], + alternative=[PuppetString(s='c')])]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + assert parse(s2) == r2 + assert ser(r2) == s2 + assert parse(s3) == r3 + # TODO elsif + # assert ser(r3) == s3 + + +def test_unless(): + s1 = """ +unless 1 { + 'a' +} + """.strip() + r1 = PuppetUnless(condition=PuppetNumber(x=1), + consequent=[PuppetString(s='a')]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_hash(): + s1 = "{}" + # TODO alignment + s2 = """ +{ + a => 1, + long_key => 'Hello', +} + """.strip() + + r1 = PuppetHash(entries=[]) + r2 = PuppetHash([HashEntry(k=PuppetQn(name='a'), + v=PuppetNumber(x=1)), + HashEntry(k=PuppetQn(name='long_key'), + v=PuppetString(s='Hello'))]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + assert parse(s2) == r2 + assert ser(r2) == s2 + + +def test_heredoc(): + s1 = r"""@("EOF"/$rt) +\$ ${x} +\r \\ +| EOF""" + + r1 = PuppetHeredoc([PuppetString(s='$ '), + PuppetVar(name='x'), + PuppetString(s='\n\r \\\n')]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_literalheredoc(): + s1 = """ +@(EOF) + Hello, ${world}" + | EOF + """.strip() + + # Test both that EOF isn't hard coded + # Note that this is slightly hacky with giving EOL, since we need + # to "know" that is how the code is implemented. + # + # Test syntax at same ttime + s2 = """ +@(EOL:txt) + EOF + |- EOL + """.strip() + + r1 = PuppetLiteralHeredoc('Hello, ${world}"\n') + r2 = PuppetLiteralHeredoc('EOF', syntax='txt') + + assert parse(s1) == r1 + assert ser(r1) == s1 + + assert parse(s2) == r2 + assert ser(r2) == s2 + + +@pytest.mark.skip(reason="When is this triggered?") +def test_parenthesis(): + # TODO + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_regex(): + s1 = "/test/" + r1 = PuppetRegex('test') + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_invoke(): + s1 = "f()" + s2 = "f(1, 2)" + s3 = "include ::example" + + r1 = PuppetInvoke(func=PuppetQn(name='f'), args=[]) + r2 = PuppetInvoke(func=PuppetQn(name='f'), + args=[PuppetNumber(x=1), PuppetNumber(x=2)]) + r3 = PuppetInvoke(func=PuppetQn(name='include'), + args=[PuppetQn(name='::example')]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + assert parse(s2) == r2 + assert ser(r2) == s2 + + assert parse(s3) == r3 + assert ser(r3) == s3 + + +def test_resourcedefaults(): + s1 = """ +File { + path => '/', +} + """.strip() + + r1 = PuppetResourceDefaults( + type=PuppetQr(name='File'), + ops=[PuppetInstanciationParameter(k='path', + v=PuppetString(s='/'), + arrow='=>')]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_resourceoverride(): + s1 = """ +File['/'] { + ensure => absent, +} + """.strip() + + r1 = PuppetResourceOverride( + resource=PuppetAccess(how=PuppetQr(name='File'), + args=[PuppetString(s='/')]), + ops=[PuppetInstanciationParameter(k='ensure', + v=PuppetQn(name='absent'), + arrow='=>')]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_declaration(): + s1 = "$x = 10" + r1 = PuppetDeclaration(k=PuppetVar(name='x'), v=PuppetNumber(x=10)) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_selector(): + s1 = """ +$test ? { + Int => 1, + Float => 2, +} + """.strip() + + r1= PuppetSelector(resource=PuppetVar(name='test'), + cases=[(PuppetQr(name='Int'), PuppetNumber(x=1)), + (PuppetQr(name='Float'), PuppetNumber(x=2))]) + + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_block(): + s1 = """ +1 +2 +""".strip() + r1 = PuppetBlock(entries=[PuppetNumber(x=1), PuppetNumber(x=2)]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_node(): + s1 = """ +node 'node.example.com' { + include profiles::example +} + """.strip() + + r1 = PuppetNode(matches=[PuppetString('node.example.com')], + body=[PuppetInvoke(func=PuppetQn('include'), + args=[PuppetQn('profiles::example')])]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_resource(): + + # TODO alignment + s1 = """ +file { '/path': + key => 'value', + * => {}, +} + """.strip() + + s2 = """ +file { + default: + mode => '0700', + ; + [ + '/a', + '/b', + ]: + user => 'root', + ; +} + """.strip() + + r1 = PuppetResource(type=PuppetQn(name='file'), + bodies=[(PuppetString(s='/path'), + [PuppetInstanciationParameter(k='key', + v=PuppetString(s='value'), + arrow='=>'), + PuppetInstanciationParameter(k='*', + v=PuppetHash(entries=[]), + arrow='=>')])]) + + r2 = PuppetResource( + type=PuppetQn(name='file'), + bodies=[(PuppetKeyword(name='default'), + [PuppetInstanciationParameter(k='mode', + v=PuppetString(s='0700'), + arrow='=>')]), + (PuppetArray(items=[PuppetString('/a'), + PuppetString('/b')]), + [PuppetInstanciationParameter(k='user', + v=PuppetString('root'), + arrow='=>')])]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + assert parse(s2) == r2 + assert ser(r2) == s2 + + +def test_define(): + s1 = """ +define a::b ( + String $x, + $y, + $z = 20, + String $w = '10', +) { + include ::something +} + """.strip() + + r1 = PuppetDefine(name='a::b', + params=[PuppetDeclarationParameter(k='x', + v=None, + type=PuppetQr(name='String')), + PuppetDeclarationParameter(k='y', v=None, type=None), + PuppetDeclarationParameter(k='z', + v=PuppetNumber(x=20), + type=None), + PuppetDeclarationParameter(k='w', + v=PuppetString(s='10'), + type=PuppetQr(name='String'))], + body=[PuppetInvoke(func=PuppetQn(name='include'), + args=[PuppetQn(name='::something')])]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_function(): + s1 = """ +function f ( + $x, +) >> String { + $x +} + """.strip() + + r1 = PuppetFunction( + name='f', + params=[PuppetDeclarationParameter(k='x')], + returns=PuppetQr(name='String'), + body=[PuppetVar(name='x')]) + + assert parse(s1) == r1 + assert ser(r1) == s1 + + +def test_class(): + s1 = """ +class name ( + String $x = 'Hello', + Int $y = 10, +) { + notice($x) +} + """.strip() + + r1 = PuppetClass(name='name', + params=[ + PuppetDeclarationParameter( + k='x', + v=PuppetString(s='Hello'), + type=PuppetQr(name='String')), + PuppetDeclarationParameter( + k='y', + v=PuppetNumber(x=10), + type=PuppetQr(name='Int')) + ], + body=[PuppetInvoke(func=PuppetQn(name='notice'), + args=[PuppetVar(name='x')])]) + + assert parse(s1) == r1 + assert ser(r1) == s1 diff --git a/tests/test_parse.py b/tests/test_parse.py deleted file mode 100644 index f7f2f68..0000000 --- a/tests/test_parse.py +++ /dev/null @@ -1,538 +0,0 @@ -""" -Unit tests for the "parser". - -TODO rename parser to "re-interpreter" (or similar), `puppet parse` is -the parser, we just rewrite the tree "slightly". -""" - -from muppet.format import parse, parse_puppet, ind, keyword -from muppet.data import tag, link, id - - -def run(s): - return parse(parse_puppet(s), 0, ['root']) - - -def s(st): - """Bulid a string literal.""" - return tag(f"'{st}'", 'literal', 'string') - - -# Helper literals, ensured correct in test_parse_literals -true = tag('true', 'literal', 'true') -false = tag('false', 'literal', 'false') -one = tag('1', 'literal', 'number') -two = tag('2', 'literal', 'number') - - -def test_parse_literals(): - # Helper literals to cut down on typing - - assert run('undef') == tag('undef', 'literal', 'undef') - assert run('true') == true - assert run('false') == false - - assert run('1') == one - assert run('2') == two - assert run('1.1') == tag('1.1', 'literal', 'number') - - assert run('default') == keyword('default') - - assert run('"Hello"') == s("Hello") - - assert run('') == tag('', 'nop') - - -def test_parse_arrays(): - assert run('[]') == tag('[]', 'array') - assert run('[1]') \ - == tag(['[', '\n', - ind(2), - tag('1', 'literal', 'number'), - ',', '\n', ind(0), - ']'], - 'array') - - -def test_parse_hash(): - - assert run('{}') == tag('{}', 'hash') - assert run(''' - { - k => 1, - v => 2, - } - ''') == \ - tag(['{', '\n', - tag([ind(1), tag('k', 'qn'), ' ', '=>', ' ', one, ',', '\n', - ind(1), tag('v', 'qn'), ' ', '=>', ' ', two, ',', '\n']), - ind(0), '}',], - 'hash') - - -def test_parse_var(): - # var - assert run('$x') == link(tag('$x', 'var'), '#x') - - -def test_parse_operators(): - - assert run('true and true') == tag([true, ' ', keyword('and'), ' ', true]) - assert run('true or true') == tag([true, ' ', keyword('or'), ' ', true]) - - # Negative literal / unary minus - assert run('- 1') == tag('-1', 'literal', 'number') - assert run('- $x') == tag(['-', ' ', link(tag('$x', 'var'), '#x')]) - - assert run('! true') == tag(['!', ' ', true]) - - assert run('1 + 2') == tag([one, ' ', '+', ' ', two]) - assert run('1 - 2') == tag([one, ' ', '-', ' ', two]) - assert run('1 * 2') == tag([one, ' ', '*', ' ', two]) - assert run('1 % 2') == tag([one, ' ', '%', ' ', two]) - assert run('1 << 2') == tag([one, ' ', '<<', ' ', two]) - assert run('1 >> 2') == tag([one, ' ', '>>', ' ', two]) - assert run('1 >= 2') == tag([one, ' ', '>=', ' ', two]) - assert run('1 <= 2') == tag([one, ' ', '<=', ' ', two]) - assert run('1 > 2') == tag([one, ' ', '>', ' ', two]) - assert run('1 < 2') == tag([one, ' ', '<', ' ', two]) - assert run('1 / 2') == tag([one, ' ', '/', ' ', two]) - assert run('1 == 2') == tag([one, ' ', '==', ' ', two]) - assert run('1 != 2') == tag([one, ' ', '!=', ' ', two]) - assert run('1 =~ 2') == tag([one, ' ', '=~', ' ', two]) - assert run('1 !~ 2') == tag([one, ' ', '!~', ' ', two]) - - assert run('1 in $x') == tag([one, ' ', - keyword('in'), - ' ', - link(tag('$x', 'var'), '#x')]) - - -def test_parse_conditionals(): - assert run('if true {}') \ - == tag([keyword('if'), ' ', true, ' ', '{', '\n', - ind(0), '}']) - - assert run('if true {} else {}') \ - == tag([keyword('if'), ' ', true, ' ', '{', '\n', - ind(0), '}']) - - # different if forms - assert run('if true {1}') \ - == tag([keyword('if'), ' ', true, ' ', '{', '\n', - ind(1), one, '\n', - ind(0), '}']) - - assert run('if true {} else {1}') \ - == tag([tag('if', 'keyword', 'if'), ' ', true, ' ', '{', '\n', - ind(0), '}', ' ', tag('else', 'keyword', 'else'), ' ', '{', '\n', - ind(1), one, '\n', - ind(0), '}']) - - assert run('if true {1} else {2}') \ - == tag([keyword('if'), ' ', true, ' ', '{', '\n', - ind(1), one, '\n', - ind(0), '}', ' ', tag('else', 'keyword', 'else'), ' ', '{', '\n', - ind(1), two, '\n', - ind(0), '}']) - - assert run('if true {1} elsif false {2}') \ - == tag([keyword('if'), ' ', true, ' ', '{', '\n', - ind(1), one, '\n', - ind(0), '}', ' ', 'els', - tag([keyword('if'), ' ', false, ' ', '{', '\n', - ind(1), two, '\n', - ind(0), '}'])]) - - assert run('if true {1} elsif false {2} else {1}') \ - == tag([keyword('if'), ' ', true, ' ', '{', '\n', - ind(1), one, '\n', - ind(0), '}', ' ', 'els', - tag([keyword('if'), ' ', false, ' ', '{', '\n', - ind(1), two, '\n', - ind(0), '}', ' ', keyword('else'), ' ', '{', '\n', - ind(1), one, '\n', - ind(0), '}'])]) - - assert run('unless true {}') \ - == tag([keyword('unless'), ' ', true, ' ', '{', '\n', - ind(0), '}']) - - assert run('unless true {1}') \ - == tag([keyword('unless'), ' ', true, ' ', '{', '\n', - ind(1), one, '\n', - ind(0), '}']) - - -def test_parse_access(): - - assert run('x[1]') == tag([tag('x', 'qn'), '[', one, ']'], - 'access') - assert run('X[1]') == tag([tag('X', 'qr'), '[', one, ']'], - 'access') - - -def test_parse_misc(): - - assert run('(1)') == tag(['(', one, ')'], 'paren') - - assert run('/hello/') == tag(['/', tag('hello', 'regex-body'), '/'], 'regex') - - assert run('File <<| |>>') \ - == tag([tag('File', 'qr'), - ' ', - tag(['<<|', ' ', '|>>'])]) - assert run("File <<| name == 'f' |>>") \ - == tag([tag('File', 'qr'), - ' ', - tag(['<<|', ' ', - tag([tag('name', 'qn'), ' ', '==', ' ', s('f')]), - ' ', '|>>'])]) - - assert run('File <| |>') \ - == tag([tag('File', 'qr'), ' ', - tag(['<|', ' ', '|>'])]) - - assert run('File <| $x |>') \ - == tag([tag('File', 'qr'), ' ', - tag(['<|', ' ', link(tag('$x', 'var'), '#x'), ' ', '|>'])]) - - -def test_parse_strings(): - # This doesn't include "atomic" strings, or regexes, since they - # are trivial types and handled above - - # Double quoted "decays" to single quoted - assert run('"hello"') == s('hello') - - assert run('"hello${x}"') \ - == tag(['"', 'hello', tag(['${', link(tag('x', 'var'), '#x'), '}'], 'str-var'), '"'], - 'string') - - assert run('"hello${1 + $x}"') \ - == tag(['"', 'hello', tag(['${', tag([one, ' ', '+', ' ', - link(tag('$x', 'var'), '#x')]), '}'], - 'str-var'), '"'], - 'string') - - # Variable like, but without interpolation - assert run(''' - @(AAA) - Hello ${x} - | AAA - ''') \ - == tag(['@(EOF)', '\n', - ind(0), 'Hello ${x}', '\n', - ind(0), '|', ' ', 'EOF'], - 'heredoc', 'literal') - - # Variable interpolation - assert run(''' - @("BBB") - Hello ${x} - | BBB - ''') \ - == tag(['@("EOF")', '\n', - ind(0), 'Hello ', tag(['${', link(tag('x', 'var'), '#x'), '}']), '\n', - ind(0), '|', ' ', 'EOF',], - 'heredoc', 'literal') - - # Variable interpolation in middle of line - assert run(''' - @("BBB") - Hello ${x}! - | BBB - ''') \ - == tag(['@("EOF")', '\n', - ind(0), 'Hello ', tag(['${', link(tag('x', 'var'), '#x'), '}']), '!', '\n', - ind(0), '|', ' ', 'EOF',], - 'heredoc', 'literal') - - # Delete trailning newline + interpolation with no variables - # also empty line - assert run(''' - @("CCC") - Hello - - World - |- CCC - ''') \ - == tag(['@(EOF)', '\n', - ind(0), 'Hello', '\n', - '\n', - ind(0), 'World', '\n', - ind(0), '|-', ' ', 'EOF',], - 'heredoc', 'literal') - - # With escape hatch ('$'), variable at end, and no trailing - # whitespace - assert run(r''' - @("DDD"/$) - \$Hello ${x} - |- DDD - ''') \ - == tag(['@("EOF")', '\n', - # Trailing newline SHALL be here. Since the newline - # stripping is done by the '|-' token - ind(0), '$Hello ', tag(['${', link(tag('x', 'var'), '#x'), '}']), '\n', - ind(0), '|-', ' ', 'EOF',], - 'heredoc', 'literal') - - # With escape hatch ('$'), variable at end, and WITH trailing - # whitespace - assert run(r''' - @("DDD"/$) - \$Hello ${x} - | DDD - ''') \ - == tag(['@("EOF")', '\n', - ind(0), '$Hello ', tag(['${', link(tag('x', 'var'), '#x'), '}']), '\n', - ind(0), '|', ' ', 'EOF',], - 'heredoc', 'literal') - - # Heredoc with variables and empty lines - assert run(''' - @("EEE") - ${x} - - ${y} - | EEE - ''') == tag(['@("EOF")', '\n', - ind(0), tag(['${', link(tag('x', 'var'), '#x'), '}']), '\n', - '\n', - ind(0), tag(['${', link(tag('y', 'var'), '#y'), '}']), '\n', - ind(0), '|', ' ', 'EOF'], - 'heredoc', 'literal') - - # Empty heredoc - assert run(''' - @(FFF) - | FFF - ''') == tag(['@(EOF)', '\n', ind(0), '|', ' ', 'EOF'], - 'heredoc', 'literal') - - # TODO Heredoc containing (generated) delimiter - - # Heredoc containing advanced expression - # This is mostly to ensure that variables get dollars again - assert run(''' - @("GGG") - ${1 + $x} - | GGG - ''') == tag(['@("EOF")', '\n', - ind(0), tag(['${', tag([one, ' ', '+', ' ', - link(tag('$x', 'var'), '#x')]), '}']), '\n', - ind(0), '|', ' ', 'EOF'], - 'heredoc', 'literal') - - -def test_declare(): - pass - assert run('$x = 1') \ - == tag([tag(id('$x', 'x'), 'var'), ' ', '=', ' ', one], 'declaration') - - -def test_parse_resource_instanciation(): - - # Single instanced resources - assert run(''' - file { 'filename': - ensure => present, - } - ''') == tag([tag('file', 'qn'), ' ', '{', ' ', s('filename'), ':', '\n', - ind(1), tag('ensure', 'parameter'), '', ' ', '=>', ' ', tag('present', 'qn'), - ',', '\n', ind(0), '}'] - ) - - assert run(''' - file { 'filename': - ensure => present, - * => {}, - } - ''') == tag([tag('file', 'qn'), ' ', '{', ' ', s('filename'), ':', '\n', - ind(1), tag('ensure', 'parameter'), '', - ' ', '=>', ' ', tag('present', 'qn'), ',', '\n', - ind(1), tag('*', 'parameter', 'splat'), ' ', - ' ', '=>', ' ', tag('{}', 'hash'), ',', '\n', - ind(0), '}'] - ) - - # multi instanced resoruces - assert run(''' - file { - 'first': ensure => present ; - 'second': k => present, * => {}, - } - ''') == tag([tag('file', 'qn'), ' ', '{', '\n', - ind(1), s('first'), ':', '\n', - ind(2), tag('ensure', 'parameter'), '', - ' ', '=>', ' ', tag('present', 'qn'), ',', '\n', - ind(1), ';', '\n', - ind(1), s('second'), ':', '\n', - ind(2), tag('k', 'parameter'), '', - ' ', '=>', ' ', tag('present', 'qn'), ',', '\n', - ind(2), tag('*', 'parameter', 'splat'), '', - ' ', '=>', ' ', tag('{}', 'hash'), ',', '\n', - ind(1), ';', '\n', - ind(0), '}', - ]) - - -def test_parse_resource_defaults(): - assert run(''' - File { - x => 1, - * => 2, - } - ''') == tag([tag('File', 'qr'), ' ', '{', '\n', - ind(1), tag('x', 'parameter'), '', ' ', '=>', ' ', one, ',', '\n', - ind(1), tag('*', 'parameter', 'splat'), '', ' ', '=>', ' ', two, ',', '\n', - ind(0), '}', - ]) - - -def test_parse_resource_override(): - assert run(''' - File['this'] { - x => 1, - y +> 2, - * => 1, - } - ''') == tag([tag([tag('File', 'qr'), '[', s('this'), ']'], 'access'), ' ', '{', '\n', - ind(1), tag('x', 'parameter'), '', ' ', '=>', ' ', one, ',', '\n', - ind(1), tag('y', 'parameter'), '', ' ', '+>', ' ', two, ',', '\n', - ind(1), tag('*', 'parameter', 'splat'), '', ' ', '=>', ' ', one, ',', '\n', - ind(0), '}', - ]) - - -def test_parse_call(): - assert run('f(1)') \ - == tag([tag('f', 'qn'), ' ', one], 'invoke') - - assert run('f(1, 2)') \ - == tag([tag('f', 'qn'), ' ', '(', one, ',', ' ', two, ')'], 'invoke') - - assert run('$x.f') \ - == tag([tag([link(tag('$x', 'var'), '#x'), '\n', - ind(0), '.', tag('f', 'qn')]), - '(', ')'], 'call-method') - - assert run('$x.f(1, 2)') \ - == tag([tag([link(tag('$x', 'var'), '#x'), '\n', - ind(0), '.', tag('f', 'qn')]), - '(', one, ',', ' ', two, ')'], 'call-method') - - assert run('f.each |$x| {$x}') \ - == tag([ - tag([tag('f', 'qn'), '\n', - ind(0), '.', tag('each', 'qn')]), - tag(['|', '$x', '|', ' ', '{', '\n', - ind(1), link(tag('$x', 'var'), '#x'), '\n', - ind(0), '}'], 'lambda')], 'call-method') - - # TODO call - - -def test_parse_arrows(): - assert run("File['x'] -> File['y']") \ - == tag([ - tag([tag('File', 'qr'), '[', s('x'), ']'], 'access'), - '\n', ind(0), '->', ' ', - tag([tag('File', 'qr'), '[', s('y'), ']'], 'access'), - ]) - assert run("File['x'] ~> File['y']") \ - == tag([ - tag([tag('File', 'qr'), '[', s('x'), ']'], 'access'), - '\n', ind(0), '~>', ' ', - tag([tag('File', 'qr'), '[', s('y'), ']'], 'access'), - ]) - - -def test_parse_resource_declaration(): - assert run('define newtype {}') \ - == tag([keyword('define'), ' ', - tag('newtype', 'name'), ' ', '{', '\n', - ind(0), '}']) - - assert run('define newtype { 1 }') \ - == tag([keyword('define'), ' ', - tag('newtype', 'name'), ' ', '{', '\n', - ind(1), one, '\n', - ind(0), '}']) - - assert run('define newtype ( $x, $y = 1, Integer $z = 2 ) { 1 }') \ - == tag([keyword('define'), ' ', - tag('newtype', 'name'), ' ', '(', '\n', - ind(1), tag(id('$x', 'x'), 'var'), ',', '\n', - ind(1), tag(id('$y', 'y'), 'var'), ' ', '=', ' ', one, ',', '\n', - ind(1), tag(tag('Integer', 'qr'), 'type'), ' ', - tag(id('$z', 'z'), 'var'), ' ', '=', ' ', two, ',', '\n', - ind(0), ')', ' ', '{', '\n', - ind(1), one, '\n', - ind(0), '}', - ]) - - -def test_parse_class_declaration(): - assert run('class cls {}') \ - == tag([keyword('class'), ' ', tag('cls', 'name'), ' ', '{', '\n', - ind(0), '}']) - - assert run('class cls ($x, $y = 1, Integer $z = 2) { $x }') \ - == tag([keyword('class'), ' ', tag('cls', 'name'), ' ', '(', '\n', - ind(1), tag('$x', 'var'), ',', '\n', - ind(1), tag('$y', 'var'), ' ', '=', ' ', one, ',', '\n', - ind(1), tag(tag('Integer', 'qr'), 'type'), ' ', tag('$z', 'var'), - ' ', '=', ' ', two, ',', '\n', - ind(0), ')', ' ', '{', '\n', - ind(1), link(tag('$x', 'var'), '#x'), '\n', - ind(0), '}']) - - -def test_parse_function_declaration(): - assert run('function fname {}') \ - == tag([keyword('function'), - ' ', 'fname', ' ', '{', '}']) - - assert run('function fname () >> String {}') \ - == tag([keyword('function'), - ' ', 'fname', ' ', '>>', ' ', tag('String', 'qr'), - ' ', '{', '}']) - - assert run('function fname ($x, $y = 1, Integer $z = 2) { $x }') \ - == tag([keyword('function'), - ' ', 'fname', ' ', '(', '\n', - ind(1), '$x', ',', '\n', - ind(1), '$y', ' ', '=', ' ', one, ',', '\n', - ind(1), tag('Integer', 'qr'), ' ', '$z', ' ', '=', ' ', two, ',', '\n', - ind(0), ')', ' ', '{', '\n', - ind(1), link(tag('$x', 'var'), '#x'), '\n', - ind(0), '}', - ]) - - -def test_parse_question_mark(): - assert run(''' - $x ? { - x => 1, - default => 2, - } - ''') == tag([link(tag('$x', 'var'), '#x'), ' ', '?', ' ', '{', '\n', - tag([ind(1), tag('x', 'qn'), ' ', '=>', ' ', one, ',', '\n', - ind(1), keyword('default'), ' ', '=>', ' ', two, ',', '\n']), - ind(0), '}'], - 'case') - - -def test_parse_case(): - assert run(''' - case true { - a: {} - } - ''') == \ - tag([keyword('case'), ' ', true, ' ', '{', '\n', - tag([ind(1), tag('a', 'qn'), ':', ' ', '{', '\n', - ind(2), tag('', 'nop'), '\n', - ind(1), '}', '\n']), - ind(0), '}']) |