From c0bd0df374e31ae6b0cb1333b225a8c0f5bd270d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20H=C3=B6rnquist?= Date: Sun, 14 Aug 2022 20:21:30 +0200 Subject: Minor cleanup. --- pyenc/__init__.py | 2 +- pyenc/app/model.py | 2 +- pyenc/enumerate_classes.py | 25 +++++++++++++++++-------- pyenc/templates/base.html | 2 +- pyenc/templates/host.html | 7 +++++++ pyenc/util.py | 5 +++++ 6 files changed, 32 insertions(+), 11 deletions(-) create mode 100644 pyenc/util.py diff --git a/pyenc/__init__.py b/pyenc/__init__.py index 90aa4fa..6400b49 100644 --- a/pyenc/__init__.py +++ b/pyenc/__init__.py @@ -76,6 +76,7 @@ def create_app(): @app.route('/environment') def environments(): + # TODO sqlalchemy order by for children? envs = model.Environment.query.all() return flask.render_template( 'list.html', @@ -117,7 +118,6 @@ def create_app(): title=name, cls=cls) - @app.route('/file') def file(): environment = request.args.get('environment') diff --git a/pyenc/app/model.py b/pyenc/app/model.py index f5b66fd..fed56f2 100644 --- a/pyenc/app/model.py +++ b/pyenc/app/model.py @@ -10,7 +10,7 @@ def init_app(app): """Adds database bindings to a Flask App.""" db.init_app(app) import logging - logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + # logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) host_classes = db.Table( diff --git a/pyenc/enumerate_classes.py b/pyenc/enumerate_classes.py index dfeb1dc..29cf2f7 100644 --- a/pyenc/enumerate_classes.py +++ b/pyenc/enumerate_classes.py @@ -114,6 +114,8 @@ def enumerate_files(path_base, environment): """ path = os.path.join(path_base, environment.name) files = find(path, type='f', name='*.pp') + files = [f for f in find(path, type='f', name='*.pp') + if os.path.basename(os.path.dirname(f)) == 'manifests'] try: for puppet_file in (model.PuppetFile(path=file) for file in files): @@ -177,28 +179,35 @@ def run(path_base: Path = '/etc/puppetlabs/code/environments', enumerate_files(path_base, environment) ### Find all puppet files which we haven't parsed - base = model.db.session \ + subexpr = model.db.session \ .query(model.PuppetFile.path, model.PuppetFile.checksum, - # Selects any of the availably environmentns. Since the checksum is the same - # the file should also be the same, regardles of which environment we chose - model.db.func.min(model.PuppetFile.environment_id)) \ + # Selects any of the availably environmentns. Since the checksum + # is the same the file should also be the same, regardles of + # which environment we chose + model.db.func.min(model.PuppetFile.environment_id).label('env_id')) \ .outerjoin(model.PuppetFileContent, model.PuppetFile.checksum == model.PuppetFileContent.checksum) \ .where(model.PuppetFileContent.json == None) \ .group_by(model.PuppetFile.checksum, - model.PuppetFile.path) + model.PuppetFile.path) \ + .cte() + + base = model.db.session \ + .query(subexpr.c.path, + subexpr.c.checksum, + model.Environment.name) \ + .join(model.Environment, + model.Environment.id == subexpr.c.env_id) files = base.all() count = base.count() - environments = {e.id: e.name for e in model.Environment.query.all()} db.session.commit() # Parse all puppet files, and store their output into pupet_file_content try: - for (i, (path, checksum, env_id)) in enumerate(files): - env = environments[env_id] + for (i, (path, checksum, env)) in enumerate(files): print(f'\x1b[2K{env} {path}') print(f'{i} / {count}', end='\r') diff --git a/pyenc/templates/base.html b/pyenc/templates/base.html index 64e35d6..b927b3d 100644 --- a/pyenc/templates/base.html +++ b/pyenc/templates/base.html @@ -30,7 +30,7 @@
diff --git a/pyenc/templates/host.html b/pyenc/templates/host.html index a44eb82..84e76e3 100644 --- a/pyenc/templates/host.html +++ b/pyenc/templates/host.html @@ -12,6 +12,13 @@ {% endfor %} + {% endblock %} {# ft:jinja #} diff --git a/pyenc/util.py b/pyenc/util.py new file mode 100644 index 0000000..df050d0 --- /dev/null +++ b/pyenc/util.py @@ -0,0 +1,5 @@ +def concatenate(listlist): + result = [] + for lst in listlist: + result.extend(lst) + return result -- cgit v1.2.3