From 7f9cd0eb580b22a2e4597c48aec65002c6f19840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20H=C3=B6rnquist?= Date: Tue, 9 Aug 2022 05:02:13 +0200 Subject: Rewrite static frontend. --- README | 20 +++++++++ pyenc/__init__.py | 97 +++++++++++++++++++++++++++++++++++----- pyenc/app/cmdline.py | 8 ++++ pyenc/app/model.py | 39 +++++++++++----- pyenc/enumerate_classes.py | 62 ++++++++++++------------- pyenc/static/css/.gitignore | 3 ++ pyenc/static/css/style.css | 43 ------------------ pyenc/static/css/style.scss | 72 +++++++++++++++++++++++++++++ pyenc/templates/base.html | 17 ++++++- pyenc/templates/class.html | 42 +++++++++++++++++ pyenc/templates/environment.html | 28 ++++++++++++ pyenc/templates/file.html | 6 +++ pyenc/templates/host.html | 17 +++++++ pyenc/templates/index.html | 6 +++ pyenc/templates/list.html | 10 +++++ requirements.txt | 37 ++++++++++++++- 16 files changed, 406 insertions(+), 101 deletions(-) create mode 100644 README create mode 100644 pyenc/static/css/.gitignore delete mode 100644 pyenc/static/css/style.css create mode 100644 pyenc/static/css/style.scss create mode 100644 pyenc/templates/class.html create mode 100644 pyenc/templates/environment.html create mode 100644 pyenc/templates/file.html create mode 100644 pyenc/templates/host.html create mode 100644 pyenc/templates/index.html create mode 100644 pyenc/templates/list.html diff --git a/README b/README new file mode 100644 index 0000000..55c2068 --- /dev/null +++ b/README @@ -0,0 +1,20 @@ +About +----- + +- Click :: For command line interfaces + +Running +------- + + `npm run build` + + `npm run watch` + + `env FLASK_APP=pyenc FLASK_ENV=development flask run --host=0.0.0.0` + + +Assets +------ +Puppet icon downloaded from +https://freesvg.org/wooden-puppet-image +Image in the public domain diff --git a/pyenc/__init__.py b/pyenc/__init__.py index c5f5ada..90aa4fa 100644 --- a/pyenc/__init__.py +++ b/pyenc/__init__.py @@ -8,15 +8,18 @@ functionallity is pulled in from other modules. import logging import random +import os.path +from domonic.html import a import flask from flask import ( - Flask, - request, - flash, - redirect, - url_for - ) + Flask, + request, + flash, + redirect, + url_for +) +from sqlalchemy.orm import joinedload from .app import model from .app import cmdline @@ -46,15 +49,85 @@ def create_app(): ]: module.init_app(app) - # not API @app.route('/') def root_page(): + # flash('Test') + return flask.render_template('index.html') + + @app.route('/host') + def hosts(): + result = model.Host.query.all() + return flask.render_template( + 'list.html', + title='Hosts', + items=[a(x.fqdn, _href=f'/host/{x.fqdn}') + for x in result]) + + @app.route('/host/') + def host(host): + host = model.Host.query.where(model.Host.fqdn == host).one() + return flask.render_template( + 'host.html', + title=host.fqdn, + env=host.environment.name, + classes=[a(x.name, _href=f'/class/{x.name}') + for x in host.classes]) + + + @app.route('/environment') + def environments(): + envs = model.Environment.query.all() + return flask.render_template( + 'list.html', + title='Environments', + items=[a(env.name, _href=f'/environment/{env.name}') + for env in envs]) + + @app.route('/environment/') + def environment(name): + env = model.Environment \ + .query \ + .where(model.Environment.name == name) \ + .one() + return flask.render_template( + 'environment.html', + title=name, + env=env) + + @app.route('/class') + def classes(): + clss = model.PuppetClass \ + .query \ + .all() + return flask.render_template( - 'start_page.html', - hosts=model.Host.query.order_by(model.Host.fqdn), - random=random, - str=str, - ) + 'list.html', + title='Classes', + items=[a(cls.name, _href=f'/class/{cls.name}') + for cls in clss]) + + @app.route('/class/') + def class_(name): + cls = model.PuppetClass \ + .query \ + .where(model.PuppetClass.name == name) \ + .options(joinedload('files').joinedload('environment')) \ + .one() + return flask.render_template('class.html', + title=name, + cls=cls) + + + @app.route('/file') + def file(): + environment = request.args.get('environment') + path = request.args.get('path') + path_base = '/var/lib/machines/busting/etc/puppetlabs/code/environments/' + with open(os.path.join(path_base, environment, path)) as f: + content = f.read() + return flask.render_template('file.html', + title=f'{environment}/{path}', + content=content) # API? @app.route('/remove', methods=['POST']) diff --git a/pyenc/app/cmdline.py b/pyenc/app/cmdline.py index 4e7e33f..101986a 100644 --- a/pyenc/app/cmdline.py +++ b/pyenc/app/cmdline.py @@ -20,6 +20,14 @@ def initialize_database(): # model.db.session.commit() +@app_group.command('drop-db') +def drop_database(): + from pyenc.app.model import db + from sqlalchemy import MetaData + m = MetaData() + m.reflect(db.engine) + m.drop_all(db.engine) + @app_group.command('enumerate-classes') @click.argument('environment') def enumerate_classes(environment): diff --git a/pyenc/app/model.py b/pyenc/app/model.py index c37cec3..f5b66fd 100644 --- a/pyenc/app/model.py +++ b/pyenc/app/model.py @@ -10,7 +10,7 @@ def init_app(app): """Adds database bindings to a Flask App.""" db.init_app(app) import logging - # logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) host_classes = db.Table( @@ -20,7 +20,6 @@ host_classes = db.Table( ) -# NOTE this is non-final, and might get removed shortly environment_classes = db.Table( 'environment_classes', db.Column('environment_id', db.ForeignKey('environment.id'), primary_key=True), @@ -88,10 +87,14 @@ class PuppetFile(db.Model): """ Puppet source code file. - Keeps track of known puppet files. Each file contains 0 to many - puppet classes. + Keeps track of known puppet files. Each file contains any number of puppet + classes. - Each file is uniquely identified by the pair (path, environment_id). + Each file optionally references a PuppetFileContent object, which contains + the (parsed) contents of the file. Multiple PuppetFile can reference the + same PuppetFileContent. + + (child) """ __tablename__ = 'puppet_file' @@ -106,8 +109,12 @@ class PuppetFile(db.Model): nullable=False) environment = db.relationship('Environment', back_populates='files') - # Checksum of the content, should be usable as a key in PuppetFileContent - checksum = db.Column(db.Text, nullable=False) + checksum = db.Column(db.Text, #db.ForeignKey('puppet_file_content.checksum'), + nullable=False) + + content = db.relationship('PuppetFileContent', + uselist=False, + primaryjoin=lambda: PuppetFile.checksum == db.foreign(PuppetFileContent.checksum)) # When we last read data into json last_parse = db.Column(db.Float) @@ -115,7 +122,6 @@ class PuppetFile(db.Model): classes = db.relationship('PuppetClass', back_populates='files', secondary=class_files) - content = db.relationship('PuppetFileContent', backref='file') __table_args__ = ( db.UniqueConstraint('path', 'environment_id'), @@ -128,15 +134,26 @@ class PuppetFileContent(db.Model): Separate from PuppetFile since many environments can share files, and I don't want to store reduntand data. + + (parent) """ __tablename__ = 'puppet_file_content' - id = db.Column(db.Integer, primary_key=True) + # id = db.Column(db.Integer, primary_key=True) - file_id = db.Column(db.Integer, db.ForeignKey(f'{PuppetFile.__tablename__}.id')) + # file_id = db.Column(db.Integer, db.ForeignKey(f'{PuppetFile.__tablename__}.id')) # Checksum of the original file - checksum = db.Column(db.Text, nullable=False) + # NOT marked as a foreign key which references + # PuppetFile.checksum, since content without a coresponding file + # is fine, and will be collected later + checksum = db.Column(db.Text, primary_key=True) + # files = db.relationship('PuppetFile', back_populates='content', + # foreign_keys=[checksum], + # primaryjoin=lambda: PuppetFile.checksum == PuppetFileContent.checksum) + + files = db.relationship('PuppetFile', + primaryjoin=lambda: PuppetFileContent.checksum == db.foreign(PuppetFile.checksum)) # Output of 'puppet parser dump --format json ' json = db.Column(db.Text, nullable=False) diff --git a/pyenc/enumerate_classes.py b/pyenc/enumerate_classes.py index c6ccf47..dfeb1dc 100644 --- a/pyenc/enumerate_classes.py +++ b/pyenc/enumerate_classes.py @@ -13,6 +13,7 @@ import os import os.path import subprocess # import time +import traceback from sqlalchemy.sql import text @@ -176,28 +177,32 @@ def run(path_base: Path = '/etc/puppetlabs/code/environments', enumerate_files(path_base, environment) ### Find all puppet files which we haven't parsed - - base = model.PuppetFile \ - .query \ - .outerjoin(model.PuppetFileContent, - model.PuppetFile.checksum == model.PuppetFileContent.checksum) \ - .where(model.PuppetFileContent.json == None) # noqa: E711 - - # count for progress bar + base = model.db.session \ + .query(model.PuppetFile.path, + model.PuppetFile.checksum, + # Selects any of the availably environmentns. Since the checksum is the same + # the file should also be the same, regardles of which environment we chose + model.db.func.min(model.PuppetFile.environment_id)) \ + .outerjoin(model.PuppetFileContent, + model.PuppetFile.checksum == model.PuppetFileContent.checksum) \ + .where(model.PuppetFileContent.json == None) \ + .group_by(model.PuppetFile.checksum, + model.PuppetFile.path) + + files = base.all() count = base.count() - - result = base \ - .join(model.Environment) \ - .add_column(model.Environment.name) \ - .all() + environments = {e.id: e.name for e in model.Environment.query.all()} db.session.commit() + + # Parse all puppet files, and store their output into pupet_file_content try: - for (i, (puppet_file, env)) in enumerate(result): - print(env, puppet_file.path) + for (i, (path, checksum, env_id)) in enumerate(files): + env = environments[env_id] + print(f'\x1b[2K{env} {path}') print(f'{i} / {count}', end='\r') - full_path = os.path.join(path_base, env, puppet_file.path) + full_path = os.path.join(path_base, env, path) try: item = puppet_parse(full_path) @@ -219,14 +224,13 @@ def run(path_base: Path = '/etc/puppetlabs/code/environments', with open(full_path, 'rb') as f: current_checksum = hashlib.sha256(f.read()).hexdigest() - if current_checksum != puppet_file.checksum: - print(f'Checksum changed for {env}/{puppet_file.path}') + if current_checksum != checksum: + print(f'Checksum changed for {env}/{path}') continue # File parsed was file we expected to parse, addit to the # database - pfc = model.PuppetFileContent(file_id=puppet_file.id, - checksum=puppet_file.checksum, + pfc = model.PuppetFileContent(checksum=checksum, json=item) db.session.add(pfc) @@ -235,15 +239,12 @@ def run(path_base: Path = '/etc/puppetlabs/code/environments', # TODO sqlite fails here, complains that the "database is locked" db.session.commit() - for file_content in model.PuppetFileContent.query.all(): + # Interpret the parsed result of all parsed puppet files + # This takes a few seconds + for file in model.PuppetFile.query.where(model.PuppetFile.content).all(): try: - class_names = interpret_file(json.loads(file_content.json)) + class_names = interpret_file(json.loads(file.content.json)) for class_name in class_names: - # cls = model.PuppetClass(class_name=class_name) - # cls.environments.append(environment) - # cls.files.append(file_content.file) - - # Add classs (if not exists) db.engine.execute(text(""" INSERT INTO puppet_class (name) VALUES (:name) @@ -262,11 +263,10 @@ def run(path_base: Path = '/etc/puppetlabs/code/environments', INSERT INTO class_files (file_id, class_id) SELECT :file, id FROM puppet_class WHERE puppet_class.name = :name ON CONFLICT (file_id, class_id) DO NOTHING - """), {'file': file_content.file_id, 'name': class_name}) - + """), {'file': file.id, 'name': class_name}) except Exception as e: - print(e) - # print(f'Failed: {puppet_file.path}') + print(f'Error for {file.id} ({file.path}) - {e}') + traceback.print_exc() db.session.commit() diff --git a/pyenc/static/css/.gitignore b/pyenc/static/css/.gitignore new file mode 100644 index 0000000..0a43926 --- /dev/null +++ b/pyenc/static/css/.gitignore @@ -0,0 +1,3 @@ +*.css +*.css.map +.sass-cache diff --git a/pyenc/static/css/style.css b/pyenc/static/css/style.css deleted file mode 100644 index 4dcef2a..0000000 --- a/pyenc/static/css/style.css +++ /dev/null @@ -1,43 +0,0 @@ -td { - padding-right: 1ex; -} - -h2 { - font-family: mono; - font-size: 14pt; -} - -dt { - font-weight: bold; -} - -dd ul { - padding-left: 0; -} - -.hosts { - display: flex; - flex-wrap: wrap; -} - -.host { - display: block; - border: 1px solid black; - padding: 1em; -} - -.flashes li { - display: block; - max-width: 100%; - padding: 2em; - background-color: yellow; - border: 2px solid orange; -} - -.added { - color: green; -} - -.removed { - color: red; -} diff --git a/pyenc/static/css/style.scss b/pyenc/static/css/style.scss new file mode 100644 index 0000000..9c4e76f --- /dev/null +++ b/pyenc/static/css/style.scss @@ -0,0 +1,72 @@ +:root { + --sidebar-h: 188; + --sidebar-s: 50%; + --sidebar-l: 50%; +} + +body { + margin: 0; + display: grid; + grid-template-areas: + "nav flash" + "nav main"; + /* sidebar width */ + grid-template-columns: 20ch auto; + /* Make flashes as small as possible */ + grid-template-rows: auto 1fr; + min-height: 100vh; +} + +#flash { + grid-area: flash; + background-color: pink; +} + +body > nav { + grid-area: nav; + background-color: hsl(var(--sidebar-h), var(--sidebar-s), var(--sidebar-l)); + display: flex; + flex-direction: column; + + hr { + flex-grow: 1; + border: none; + } + + img { + max-width: 100%; + } + + ul { + padding: 0; + margin: 0; + } + + li { + display: block; + } + + a { + display: block; + color: initial; + text-decoration: none; + padding-bottom: 1em; + padding-top: 1em; + padding-left: 1em; + } + + a:hover { + background-color: hsl(var(--sidebar-h), calc(var(--sidebar-s) - 20%), var(--sidebar-l)); + } +} + +body > main { + grid-area: main; + background-color: beige; + padding: 1em; +} + + +table { + font-size: 80%; +} diff --git a/pyenc/templates/base.html b/pyenc/templates/base.html index 9256734..64e35d6 100644 --- a/pyenc/templates/base.html +++ b/pyenc/templates/base.html @@ -4,9 +4,10 @@ + - - Puppet Node Classifier + + Puppet Classifier
@@ -20,6 +21,18 @@ {% endif %} {% endwith %}
+
{% block content %} {% endblock %} diff --git a/pyenc/templates/class.html b/pyenc/templates/class.html new file mode 100644 index 0000000..bfae1b2 --- /dev/null +++ b/pyenc/templates/class.html @@ -0,0 +1,42 @@ +{% extends "base.html" %} +{% block content %} +

Class ‘{{ title }}’

+{{ env }} +
+
Environments
+
+
    + {% for env in cls.environments %} +
  • {{ env.name }}
  • + {% endfor %} +
+
+
Files
+
+ + + + + + + + {% for file in cls.files %} + + + + + + {% endfor %} + +
EnvironmentPathChecksum
{{ file.environment.name }}{{ file.path }}{{ file.checksum }}
+
Hosts
+
+ +
+
+{% endblock %} +{# ft:jinja #} diff --git a/pyenc/templates/environment.html b/pyenc/templates/environment.html new file mode 100644 index 0000000..b0995d6 --- /dev/null +++ b/pyenc/templates/environment.html @@ -0,0 +1,28 @@ +{% extends "base.html" %} +{% block content %} +

Environment ‘{{ title }}’

+
+
Hosts
+
+ +
+
Classes
+
+ +
+
Files
+
+
    +
+
+
+{% endblock %} +{# ft:jinja #} diff --git a/pyenc/templates/file.html b/pyenc/templates/file.html new file mode 100644 index 0000000..19bc811 --- /dev/null +++ b/pyenc/templates/file.html @@ -0,0 +1,6 @@ +{% extends "base.html" %} +{% block content %} +

File ‘{{ title }}’

+
{{ content }}
+{% endblock %} +{# ft:jinja #} diff --git a/pyenc/templates/host.html b/pyenc/templates/host.html new file mode 100644 index 0000000..a44eb82 --- /dev/null +++ b/pyenc/templates/host.html @@ -0,0 +1,17 @@ +{% extends "base.html" %} +{% block content %} +

Host ‘{{ title }}’

+
+
Environment
+
{{ env }}
+
Direct Classes
+
+
    + {% for cls in classes %} +
  • {{ cls|safe }}
  • + {% endfor %} +
+
+
+{% endblock %} +{# ft:jinja #} diff --git a/pyenc/templates/index.html b/pyenc/templates/index.html new file mode 100644 index 0000000..c10449b --- /dev/null +++ b/pyenc/templates/index.html @@ -0,0 +1,6 @@ +{% extends "base.html" %} +{% block content %} +

Puppet Classifier

+

Something might show up here if you click the sidebar...

+{% endblock %} +{# ft:jinja #} diff --git a/pyenc/templates/list.html b/pyenc/templates/list.html new file mode 100644 index 0000000..dd267c5 --- /dev/null +++ b/pyenc/templates/list.html @@ -0,0 +1,10 @@ +{% extends "base.html" %} +{% block content %} +

{{ title }}

+
    + {% for item in items %} +
  • {{ item|safe }}
  • + {% endfor %} +
+{% endblock %} +{# ft:jinja #} diff --git a/requirements.txt b/requirements.txt index f807e52..e4620f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,54 @@ +astroid==2.11.6 +attrs==22.1.0 certifi==2021.10.8 charset-normalizer==2.0.12 click==8.0.4 +cssselect==1.1.0 Deprecated==1.2.13 +dill==0.3.5.1 +domonic==0.9.11 +elementpath==2.5.3 Flask==2.0.3 +flask-openapi3==1.1.4 Flask-SQLAlchemy==2.5.1 greenlet==1.1.2 +html5lib==1.1 idna==3.3 +iniconfig==1.1.1 +isort==5.10.1 itsdangerous==2.1.0 Jinja2==3.0.3 +lazy-object-proxy==1.7.1 MarkupSafe==2.1.0 +mccabe==0.7.0 +mypy==0.961 +mypy-extensions==0.4.3 packaging==21.3 +platformdirs==2.5.2 +pluggy==1.0.0 +psycopg2==2.9.3 +py==1.11.0 py-redis==1.1.1 +pydantic==1.9.1 +pylint==2.14.3 +pylint-flask-sqlalchemy==0.2.0 pyparsing==3.0.7 +pytest==7.1.2 +python-dateutil==2.8.2 +pytoolconfig==1.2.2 PyYAML==6.0 redis==4.1.4 -requests==2.27.1 +requests==2.28.1 +rope==1.3.0 +ropemode==0.6.1 +ropevim==0.8.1 +six==1.16.0 SQLAlchemy==1.4.31 -urllib3==1.26.8 +sqlalchemy2-stubs==0.0.2a24 +tomli==2.0.1 +tomlkit==0.11.0 +typing_extensions==4.2.0 +urllib3==1.26.11 +webencodings==0.5.1 Werkzeug==2.0.3 wrapt==1.13.3 -- cgit v1.2.3