diff options
Diffstat (limited to 'pyenc/enumerate_classes.py')
| -rw-r--r-- | pyenc/enumerate_classes.py | 62 |
1 files changed, 31 insertions, 31 deletions
diff --git a/pyenc/enumerate_classes.py b/pyenc/enumerate_classes.py index c6ccf47..dfeb1dc 100644 --- a/pyenc/enumerate_classes.py +++ b/pyenc/enumerate_classes.py @@ -13,6 +13,7 @@ import os import os.path import subprocess # import time +import traceback from sqlalchemy.sql import text @@ -176,28 +177,32 @@ def run(path_base: Path = '/etc/puppetlabs/code/environments', enumerate_files(path_base, environment) ### Find all puppet files which we haven't parsed - - base = model.PuppetFile \ - .query \ - .outerjoin(model.PuppetFileContent, - model.PuppetFile.checksum == model.PuppetFileContent.checksum) \ - .where(model.PuppetFileContent.json == None) # noqa: E711 - - # count for progress bar + base = model.db.session \ + .query(model.PuppetFile.path, + model.PuppetFile.checksum, + # Selects any of the availably environmentns. Since the checksum is the same + # the file should also be the same, regardles of which environment we chose + model.db.func.min(model.PuppetFile.environment_id)) \ + .outerjoin(model.PuppetFileContent, + model.PuppetFile.checksum == model.PuppetFileContent.checksum) \ + .where(model.PuppetFileContent.json == None) \ + .group_by(model.PuppetFile.checksum, + model.PuppetFile.path) + + files = base.all() count = base.count() - - result = base \ - .join(model.Environment) \ - .add_column(model.Environment.name) \ - .all() + environments = {e.id: e.name for e in model.Environment.query.all()} db.session.commit() + + # Parse all puppet files, and store their output into pupet_file_content try: - for (i, (puppet_file, env)) in enumerate(result): - print(env, puppet_file.path) + for (i, (path, checksum, env_id)) in enumerate(files): + env = environments[env_id] + print(f'\x1b[2K{env} {path}') print(f'{i} / {count}', end='\r') - full_path = os.path.join(path_base, env, puppet_file.path) + full_path = os.path.join(path_base, env, path) try: item = puppet_parse(full_path) @@ -219,14 +224,13 @@ def run(path_base: Path = '/etc/puppetlabs/code/environments', with open(full_path, 'rb') as f: current_checksum = hashlib.sha256(f.read()).hexdigest() - if current_checksum != puppet_file.checksum: - print(f'Checksum changed for {env}/{puppet_file.path}') + if current_checksum != checksum: + print(f'Checksum changed for {env}/{path}') continue # File parsed was file we expected to parse, addit to the # database - pfc = model.PuppetFileContent(file_id=puppet_file.id, - checksum=puppet_file.checksum, + pfc = model.PuppetFileContent(checksum=checksum, json=item) db.session.add(pfc) @@ -235,15 +239,12 @@ def run(path_base: Path = '/etc/puppetlabs/code/environments', # TODO sqlite fails here, complains that the "database is locked" db.session.commit() - for file_content in model.PuppetFileContent.query.all(): + # Interpret the parsed result of all parsed puppet files + # This takes a few seconds + for file in model.PuppetFile.query.where(model.PuppetFile.content).all(): try: - class_names = interpret_file(json.loads(file_content.json)) + class_names = interpret_file(json.loads(file.content.json)) for class_name in class_names: - # cls = model.PuppetClass(class_name=class_name) - # cls.environments.append(environment) - # cls.files.append(file_content.file) - - # Add classs (if not exists) db.engine.execute(text(""" INSERT INTO puppet_class (name) VALUES (:name) @@ -262,11 +263,10 @@ def run(path_base: Path = '/etc/puppetlabs/code/environments', INSERT INTO class_files (file_id, class_id) SELECT :file, id FROM puppet_class WHERE puppet_class.name = :name ON CONFLICT (file_id, class_id) DO NOTHING - """), {'file': file_content.file_id, 'name': class_name}) - + """), {'file': file.id, 'name': class_name}) except Exception as e: - print(e) - # print(f'Failed: {puppet_file.path}') + print(f'Error for {file.id} ({file.path}) - {e}') + traceback.print_exc() db.session.commit() |