Rewrite static frontend.

16 files changed, 406 insertions, 101 deletions

diff --git a/README b/README
new file mode 100644
index 0000000..55c2068
--- /dev/null
+++ b/README
@@ -0,0 +1,20 @@
+About
+-----
+
+- Click :: For command line interfaces
+
+Running
+-------
+
+	`npm run build`
+
+	`npm run watch`
+
+	`env FLASK_APP=pyenc FLASK_ENV=development flask run --host=0.0.0.0`
+
+
+Assets
+------
+Puppet icon downloaded from
+https://freesvg.org/wooden-puppet-image
+Image in the public domain
diff --git a/pyenc/__init__.py b/pyenc/__init__.py
index c5f5ada..90aa4fa 100644
--- a/pyenc/__init__.py
+++ b/pyenc/__init__.py
@@ -8,15 +8,18 @@ functionallity is pulled in from other modules.
 
 import logging
 import random
+import os.path
 
+from domonic.html import a
 import flask
 from flask import (
-        Flask,
-        request,
-        flash,
-        redirect,
-        url_for
-        )
+    Flask,
+    request,
+    flash,
+    redirect,
+    url_for
+)
+from sqlalchemy.orm import joinedload
 
 from .app import model
 from .app import cmdline
@@ -46,15 +49,85 @@ def create_app():
     ]:
         module.init_app(app)
 
-    # not API
     @app.route('/')
     def root_page():
+        # flash('Test')
+        return flask.render_template('index.html')
+
+    @app.route('/host')
+    def hosts():
+        result = model.Host.query.all()
+        return flask.render_template(
+            'list.html',
+            title='Hosts',
+            items=[a(x.fqdn, _href=f'/host/{x.fqdn}')
+                   for x in result])
+
+    @app.route('/host/<host>')
+    def host(host):
+        host = model.Host.query.where(model.Host.fqdn == host).one()
+        return flask.render_template(
+            'host.html',
+            title=host.fqdn,
+            env=host.environment.name,
+            classes=[a(x.name, _href=f'/class/{x.name}')
+                     for x in host.classes])
+
+
+    @app.route('/environment')
+    def environments():
+        envs = model.Environment.query.all()
+        return flask.render_template(
+            'list.html',
+            title='Environments',
+            items=[a(env.name, _href=f'/environment/{env.name}')
+                   for env in envs])
+
+    @app.route('/environment/<name>')
+    def environment(name):
+        env = model.Environment \
+                   .query \
+                   .where(model.Environment.name == name) \
+                   .one()
+        return flask.render_template(
+            'environment.html',
+            title=name,
+            env=env)
+
+    @app.route('/class')
+    def classes():
+        clss = model.PuppetClass \
+                    .query \
+                    .all()
+
         return flask.render_template(
-                'start_page.html',
-                hosts=model.Host.query.order_by(model.Host.fqdn),
-                random=random,
-                str=str,
-                )
+            'list.html',
+            title='Classes',
+            items=[a(cls.name, _href=f'/class/{cls.name}')
+                   for cls in clss])
+
+    @app.route('/class/<name>')
+    def class_(name):
+        cls = model.PuppetClass \
+                   .query \
+                   .where(model.PuppetClass.name == name) \
+                   .options(joinedload('files').joinedload('environment')) \
+                   .one()
+        return flask.render_template('class.html',
+                                     title=name,
+                                     cls=cls)
+
+
+    @app.route('/file')
+    def file():
+        environment = request.args.get('environment')
+        path = request.args.get('path')
+        path_base = '/var/lib/machines/busting/etc/puppetlabs/code/environments/'
+        with open(os.path.join(path_base, environment, path)) as f:
+            content = f.read()
+        return flask.render_template('file.html',
+                                     title=f'{environment}/{path}',
+                                     content=content)
 
     # API?
     @app.route('/remove', methods=['POST'])
diff --git a/pyenc/app/cmdline.py b/pyenc/app/cmdline.py
index 4e7e33f..101986a 100644
--- a/pyenc/app/cmdline.py
+++ b/pyenc/app/cmdline.py
@@ -20,6 +20,14 @@ def initialize_database():
     # model.db.session.commit()
 
 
+@app_group.command('drop-db')
+def drop_database():
+    from pyenc.app.model import db
+    from sqlalchemy import MetaData
+    m = MetaData()
+    m.reflect(db.engine)
+    m.drop_all(db.engine)
+
 @app_group.command('enumerate-classes')
 @click.argument('environment')
 def enumerate_classes(environment):
diff --git a/pyenc/app/model.py b/pyenc/app/model.py
index c37cec3..f5b66fd 100644
--- a/pyenc/app/model.py
+++ b/pyenc/app/model.py
@@ -10,7 +10,7 @@ def init_app(app):
     """Adds database bindings to a Flask App."""
     db.init_app(app)
     import logging
-    # logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
+    logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
 
 
 host_classes = db.Table(
@@ -20,7 +20,6 @@ host_classes = db.Table(
 )
 
 
-# NOTE this is non-final, and might get removed shortly
 environment_classes = db.Table(
     'environment_classes',
     db.Column('environment_id', db.ForeignKey('environment.id'), primary_key=True),
@@ -88,10 +87,14 @@ class PuppetFile(db.Model):
     """
     Puppet source code file.
 
-    Keeps track of known puppet files. Each file contains 0 to many
-    puppet classes.
+    Keeps track of known puppet files. Each file contains any number of puppet
+    classes.
 
-    Each file is uniquely identified by the pair (path, environment_id).
+    Each file optionally references a PuppetFileContent object, which contains
+    the (parsed) contents of the file. Multiple PuppetFile can reference the
+    same PuppetFileContent.
+
+    (child)
     """
 
     __tablename__ = 'puppet_file'
@@ -106,8 +109,12 @@ class PuppetFile(db.Model):
                                nullable=False)
     environment = db.relationship('Environment', back_populates='files')
 
-    # Checksum of the content, should be usable as a key in PuppetFileContent
-    checksum = db.Column(db.Text, nullable=False)
+    checksum = db.Column(db.Text, #db.ForeignKey('puppet_file_content.checksum'),
+                         nullable=False)
+
+    content = db.relationship('PuppetFileContent',
+                              uselist=False,
+                              primaryjoin=lambda: PuppetFile.checksum == db.foreign(PuppetFileContent.checksum))
 
     # When we last read data into json
     last_parse = db.Column(db.Float)
@@ -115,7 +122,6 @@ class PuppetFile(db.Model):
     classes = db.relationship('PuppetClass',
                               back_populates='files',
                               secondary=class_files)
-    content = db.relationship('PuppetFileContent', backref='file')
 
     __table_args__ = (
         db.UniqueConstraint('path', 'environment_id'),
@@ -128,15 +134,26 @@ class PuppetFileContent(db.Model):
 
     Separate from PuppetFile since many environments can share files,
     and I don't want to store reduntand data.
+
+    (parent)
     """
     __tablename__ = 'puppet_file_content'
 
-    id = db.Column(db.Integer, primary_key=True)
+    # id = db.Column(db.Integer, primary_key=True)
 
-    file_id = db.Column(db.Integer, db.ForeignKey(f'{PuppetFile.__tablename__}.id'))
+    # file_id = db.Column(db.Integer, db.ForeignKey(f'{PuppetFile.__tablename__}.id'))
 
     # Checksum of the original file
-    checksum = db.Column(db.Text, nullable=False)
+    # NOT marked as a foreign key which references
+    # PuppetFile.checksum, since content without a coresponding file
+    # is fine, and will be collected later
+    checksum = db.Column(db.Text, primary_key=True)
+    # files = db.relationship('PuppetFile', back_populates='content',
+    #                         foreign_keys=[checksum],
+    #                         primaryjoin=lambda: PuppetFile.checksum == PuppetFileContent.checksum)
+
+    files = db.relationship('PuppetFile',
+                            primaryjoin=lambda: PuppetFileContent.checksum == db.foreign(PuppetFile.checksum))
 
     # Output of 'puppet parser dump --format json <filename>'
     json = db.Column(db.Text, nullable=False)
diff --git a/pyenc/enumerate_classes.py b/pyenc/enumerate_classes.py
index c6ccf47..dfeb1dc 100644
--- a/pyenc/enumerate_classes.py
+++ b/pyenc/enumerate_classes.py
@@ -13,6 +13,7 @@ import os
 import os.path
 import subprocess
 # import time
+import traceback
 
 from sqlalchemy.sql import text
 
@@ -176,28 +177,32 @@ def run(path_base: Path = '/etc/puppetlabs/code/environments',
     enumerate_files(path_base, environment)
 
     ### Find all puppet files which we haven't parsed
-
-    base = model.PuppetFile \
-        .query \
-        .outerjoin(model.PuppetFileContent,
-                   model.PuppetFile.checksum == model.PuppetFileContent.checksum) \
-        .where(model.PuppetFileContent.json == None)  # noqa: E711
-
-    # count for progress bar
+    base = model.db.session \
+                   .query(model.PuppetFile.path,
+                          model.PuppetFile.checksum,
+                          # Selects any of the availably environmentns. Since the checksum is the same
+                          # the file should also be the same, regardles of which environment we chose
+                          model.db.func.min(model.PuppetFile.environment_id)) \
+                   .outerjoin(model.PuppetFileContent,
+                              model.PuppetFile.checksum == model.PuppetFileContent.checksum) \
+                   .where(model.PuppetFileContent.json == None) \
+                   .group_by(model.PuppetFile.checksum,
+                             model.PuppetFile.path)
+
+    files = base.all()
     count = base.count()
-
-    result = base \
-        .join(model.Environment) \
-        .add_column(model.Environment.name) \
-        .all()
+    environments = {e.id: e.name for e in model.Environment.query.all()}
 
     db.session.commit()
+
+    # Parse all puppet files, and store their output into pupet_file_content
     try:
-        for (i, (puppet_file, env)) in enumerate(result):
-            print(env, puppet_file.path)
+        for (i, (path, checksum, env_id)) in enumerate(files):
+            env = environments[env_id]
+            print(f'\x1b[2K{env} {path}')
             print(f'{i} / {count}', end='\r')
 
-            full_path = os.path.join(path_base, env, puppet_file.path)
+            full_path = os.path.join(path_base, env, path)
 
             try:
                 item = puppet_parse(full_path)
@@ -219,14 +224,13 @@ def run(path_base: Path = '/etc/puppetlabs/code/environments',
             with open(full_path, 'rb') as f:
                 current_checksum = hashlib.sha256(f.read()).hexdigest()
 
-            if current_checksum != puppet_file.checksum:
-                print(f'Checksum changed for {env}/{puppet_file.path}')
+            if current_checksum != checksum:
+                print(f'Checksum changed for {env}/{path}')
                 continue
 
             # File parsed was file we expected to parse, addit to the
             # database
-            pfc = model.PuppetFileContent(file_id=puppet_file.id,
-                                          checksum=puppet_file.checksum,
+            pfc = model.PuppetFileContent(checksum=checksum,
                                           json=item)
             db.session.add(pfc)
 
@@ -235,15 +239,12 @@ def run(path_base: Path = '/etc/puppetlabs/code/environments',
         # TODO sqlite fails here, complains that the "database is locked"
         db.session.commit()
 
-    for file_content in model.PuppetFileContent.query.all():
+    # Interpret the parsed result of all parsed puppet files
+    # This takes a few seconds
+    for file in model.PuppetFile.query.where(model.PuppetFile.content).all():
         try:
-            class_names = interpret_file(json.loads(file_content.json))
+            class_names = interpret_file(json.loads(file.content.json))
             for class_name in class_names:
-                # cls = model.PuppetClass(class_name=class_name)
-                # cls.environments.append(environment)
-                # cls.files.append(file_content.file)
-
-                # Add classs (if not exists)
                 db.engine.execute(text("""
                 INSERT INTO puppet_class (name)
                 VALUES (:name)
@@ -262,11 +263,10 @@ def run(path_base: Path = '/etc/puppetlabs/code/environments',
                 INSERT INTO class_files (file_id, class_id)
                 SELECT :file, id FROM puppet_class WHERE puppet_class.name = :name
                 ON CONFLICT (file_id, class_id) DO NOTHING
-                """), {'file': file_content.file_id, 'name': class_name})
-
+                """), {'file': file.id, 'name': class_name})
         except Exception as e:
-            print(e)
-            # print(f'Failed: {puppet_file.path}')
+            print(f'Error for {file.id} ({file.path}) - {e}')
+            traceback.print_exc()
 
     db.session.commit()
 
diff --git a/pyenc/static/css/.gitignore b/pyenc/static/css/.gitignore
new file mode 100644
index 0000000..0a43926
--- /dev/null
+++ b/pyenc/static/css/.gitignore
@@ -0,0 +1,3 @@
+*.css
+*.css.map
+.sass-cache
diff --git a/pyenc/static/css/style.css b/pyenc/static/css/style.css
deleted file mode 100644
index 4dcef2a..0000000
--- a/pyenc/static/css/style.css
+++ /dev/null
@@ -1,43 +0,0 @@
-td {
-	padding-right: 1ex;
-}
-
-h2 {
-	font-family: mono;
-	font-size: 14pt;
-}
-
-dt {
-	font-weight: bold;
-}
-
-dd ul {
-	padding-left: 0;
-}
-
-.hosts {
-	display: flex;
-	flex-wrap: wrap;
-}
-
-.host {
-	display: block;
-	border: 1px solid black;
-	padding: 1em;
-}
-
-.flashes li {
-	display: block;
-	max-width: 100%;
-	padding: 2em;
-	background-color: yellow;
-	border: 2px solid orange;
-}
-
-.added {
-	color: green;
-}
-
-.removed {
-	color: red;
-}
diff --git a/pyenc/static/css/style.scss b/pyenc/static/css/style.scss
new file mode 100644
index 0000000..9c4e76f
--- /dev/null
+++ b/pyenc/static/css/style.scss
@@ -0,0 +1,72 @@
+:root {
+	--sidebar-h: 188;
+	--sidebar-s: 50%;
+	--sidebar-l: 50%;
+}
+
+body {
+	margin: 0;
+	display: grid;
+	grid-template-areas:
+		"nav flash"
+		"nav main";
+	/* sidebar width */
+	grid-template-columns: 20ch auto;
+	/* Make flashes as small as possible */
+	grid-template-rows: auto 1fr;
+	min-height: 100vh;
+}
+
+#flash {
+	grid-area: flash;
+	background-color: pink;
+}
+
+body > nav {
+	grid-area: nav;
+	background-color: hsl(var(--sidebar-h), var(--sidebar-s), var(--sidebar-l));
+	display: flex;
+	flex-direction: column;
+
+	hr {
+		flex-grow: 1;
+		border: none;
+	}
+
+	img {
+		max-width: 100%;
+	}
+
+	ul {
+		padding: 0;
+		margin: 0;
+	}
+
+	li {
+		display: block;
+	}
+
+	a {
+		display: block;
+		color: initial;
+		text-decoration: none;
+		padding-bottom: 1em;
+		padding-top: 1em;
+		padding-left: 1em;
+	}
+
+	a:hover {
+		background-color: hsl(var(--sidebar-h), calc(var(--sidebar-s) - 20%), var(--sidebar-l));
+	}
+}
+
+body > main {
+	grid-area: main;
+	background-color: beige;
+	padding: 1em;
+}
+
+
+table {
+	font-size: 80%;
+}
diff --git a/pyenc/templates/base.html b/pyenc/templates/base.html
index 9256734..64e35d6 100644
--- a/pyenc/templates/base.html
+++ b/pyenc/templates/base.html
@@ -4,9 +4,10 @@
 		<meta charset="UTF-8"/>
 		<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
 		<meta name="viewport" content="width=device-width, intial-scale=1"/>
+		<link rel="icon" href="{{ url_for('static', filename='icon.svg') }}"/>
 		<link href="{{ url_for('static', filename='css/style.css') }}" rel="stylesheet"/>
-		<script type="module" src="{{ url_for('static', filename='js/script.js') }}"></script>
-		<title>Puppet Node Classifier</title>
+		<!-- <script type="module" src="{{ url_for('static', filename='js/script.js') }}"></script> -->
+		<title>Puppet Classifier</title>
 	</head>
 	<body>
 		<div id="flash">
@@ -20,6 +21,18 @@
 				{% endif %}
 			{% endwith %}
 		</div>
+		<nav>
+			<img src="{{ url_for('static', filename='icon.svg') }}"/>
+			<ul>
+				<li><a href="/host">Hosts</a>
+				<li><a href="/environment">Environments</a>
+				<li><a href="/class">Classes</a>
+			</ul>
+			<hr/>
+			<ul>
+				<li><a href="#">Source</a>
+			</ul>
+		</nav>
 		<main>
 			{% block content %}
 			{% endblock %}
diff --git a/pyenc/templates/class.html b/pyenc/templates/class.html
new file mode 100644
index 0000000..bfae1b2
--- /dev/null
+++ b/pyenc/templates/class.html
@@ -0,0 +1,42 @@
+{% extends "base.html" %}
+{% block content %}
+<h1>Class ‘{{ title }}’</h1>
+<a href="/environment/{{ env }}">{{ env }}</a>
+<dl>
+	<dt>Environments</dt>
+	<dd>
+		<ul>
+			{% for env in cls.environments %}
+			<li><a href="/environment/{{env.name}}">{{ env.name }}</a></li>
+			{% endfor %}
+		</ul>
+	</dd>
+	<dt>Files</dt>
+	<dd>
+		<table>
+			<thead>
+				<th>Environment</th>
+				<th>Path</th>
+				<th>Checksum</th>
+			</thead>
+			<tbody>
+				{% for file in cls.files %}
+				<tr>
+					<td>{{ file.environment.name }}</td>
+					<td><a href="/file?environment={{ file.environment.name }}&path={{ file.path }}">{{ file.path }}</a></td>
+					<td><code>{{ file.checksum }}</code></td>
+				</tr>
+				{% endfor %}
+			</tbody>
+		</table>
+		<dt>Hosts</dt>
+	<dd>
+		<ul>
+			{% for host in cls.hosts %}
+			<li><a href="/host/{{ host.fqdn }}">{{ host.fqdn }}</a></li>
+			{% endfor %}
+		</ul>
+	</dd>
+</dl>
+{% endblock %}
+{# ft:jinja #}
diff --git a/pyenc/templates/environment.html b/pyenc/templates/environment.html
new file mode 100644
index 0000000..b0995d6
--- /dev/null
+++ b/pyenc/templates/environment.html
@@ -0,0 +1,28 @@
+{% extends "base.html" %}
+{% block content %}
+<h1>Environment ‘{{ title }}’</h1>
+<dl>
+	<dt>Hosts</dt>
+	<dd>
+		<ul>
+			{% for host in env.hosts %}
+			<li><a href="/host/{{ host.fqdn }}">{{ host.fqdn }}</a></li>
+			{% endfor %}
+		</ul>
+	</dd>
+	<dt>Classes</dt>
+	<dd>
+		<ul>
+			{% for cls in env.classes %}
+			<li><a href="/class/{{ cls.name }}">{{ cls.name }}</a></li>
+			{% endfor %}
+		</ul>
+	</dd>
+	<dt>Files</dt>
+	<dd>
+		<ul>
+		</ul>
+	</dd>
+</dl>
+{% endblock %}
+{# ft:jinja #}
diff --git a/pyenc/templates/file.html b/pyenc/templates/file.html
new file mode 100644
index 0000000..19bc811
--- /dev/null
+++ b/pyenc/templates/file.html
@@ -0,0 +1,6 @@
+{% extends "base.html" %}
+{% block content %}
+<h1>File ‘{{ title }}’</h1>
+<pre>{{ content }}</pre>
+{% endblock %}
+{# ft:jinja #}
diff --git a/pyenc/templates/host.html b/pyenc/templates/host.html
new file mode 100644
index 0000000..a44eb82
--- /dev/null
+++ b/pyenc/templates/host.html
@@ -0,0 +1,17 @@
+{% extends "base.html" %}
+{% block content %}
+<h1>Host ‘{{ title }}’</h1>
+<dl>
+	<dt>Environment</dt>
+	<dd><a href="/environment/{{ env }}">{{ env }}</a></dd>
+	<dt>Direct Classes</dt>
+	<dd>
+		<ul>
+			{% for cls in classes %}
+			<li>{{ cls|safe }}</li>
+			{% endfor %}
+		</ul>
+	</dd>
+</dl>
+{% endblock %}
+{# ft:jinja #}
diff --git a/pyenc/templates/index.html b/pyenc/templates/index.html
new file mode 100644
index 0000000..c10449b
--- /dev/null
+++ b/pyenc/templates/index.html
@@ -0,0 +1,6 @@
+{% extends "base.html" %}
+{% block content %}
+<h1>Puppet Classifier</h1>
+<p>Something might show up here if you click the sidebar...</p>
+{% endblock %}
+{# ft:jinja #}
diff --git a/pyenc/templates/list.html b/pyenc/templates/list.html
new file mode 100644
index 0000000..dd267c5
--- /dev/null
+++ b/pyenc/templates/list.html
@@ -0,0 +1,10 @@
+{% extends "base.html" %}
+{% block content %}
+<h1>{{ title }}</h1>
+<ul>
+	{% for item in items %}
+	<li>{{ item|safe }}</li>
+	{% endfor %}
+</ul>
+{% endblock %}
+{# ft:jinja #}
diff --git a/requirements.txt b/requirements.txt
index f807e52..e4620f2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,21 +1,54 @@
+astroid==2.11.6
+attrs==22.1.0
 certifi==2021.10.8
 charset-normalizer==2.0.12
 click==8.0.4
+cssselect==1.1.0
 Deprecated==1.2.13
+dill==0.3.5.1
+domonic==0.9.11
+elementpath==2.5.3
 Flask==2.0.3
+flask-openapi3==1.1.4
 Flask-SQLAlchemy==2.5.1
 greenlet==1.1.2
+html5lib==1.1
 idna==3.3
+iniconfig==1.1.1
+isort==5.10.1
 itsdangerous==2.1.0
 Jinja2==3.0.3
+lazy-object-proxy==1.7.1
 MarkupSafe==2.1.0
+mccabe==0.7.0
+mypy==0.961
+mypy-extensions==0.4.3
 packaging==21.3
+platformdirs==2.5.2
+pluggy==1.0.0
+psycopg2==2.9.3
+py==1.11.0
 py-redis==1.1.1
+pydantic==1.9.1
+pylint==2.14.3
+pylint-flask-sqlalchemy==0.2.0
 pyparsing==3.0.7
+pytest==7.1.2
+python-dateutil==2.8.2
+pytoolconfig==1.2.2
 PyYAML==6.0
 redis==4.1.4
-requests==2.27.1
+requests==2.28.1
+rope==1.3.0
+ropemode==0.6.1
+ropevim==0.8.1
+six==1.16.0
 SQLAlchemy==1.4.31
-urllib3==1.26.8
+sqlalchemy2-stubs==0.0.2a24
+tomli==2.0.1
+tomlkit==0.11.0
+typing_extensions==4.2.0
+urllib3==1.26.11
+webencodings==0.5.1
 Werkzeug==2.0.3
 wrapt==1.13.3