1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
|
#!/usr/bin/env python3
"""
Loads all puppet files in environment, parse them, and store the
parsed data to redis.
Later run commit_classes to save them permanently.
"""
import subprocess
import json
import os
import time
import pyenc
from pyenc.db import db
import pyenc.model as model
def find(path, **kvs):
"""Wrapper around find(1)."""
cmdline = ['find', path]
for k, v in kvs.items():
cmdline.append(f'-{k}')
cmdline.append(v)
cmdline.append('-print0')
cmd = subprocess.run(cmdline, capture_output=True)
return (f for f in cmd.stdout.split(b'\0') if f)
def parse_files(files):
for i, file in enumerate(files):
st = os.stat(file)
last_modify = st.st_mtime
old_object = model.PuppetFile.query \
.where(model.PuppetFile.path == file) \
.first()
if old_object and old_object.last_parse > last_modify:
# file unchanged since our last parse, skip
continue
print(f'{i}/{len(files)}: {file}')
cmd = subprocess.Popen(
['puppet', 'parser', 'dump', '--format', 'json', file],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if cmd.returncode and cmd.returncode != 0:
print("Parsing failed")
print(cmd.returncode)
print(cmd.stderr.read())
else:
if old_object:
m = old_object
else:
m = model.PuppetFile(path=file)
m.last_parse = time.time()
m.json = cmd.stdout.read()
if cmd.wait() != 0:
print("Parsing failed (late version)")
print(cmd.stderr.read().decode('UTF-8'))
continue
db.session.add(m)
db.session.commit()
def main():
app = pyenc.create_app()
app.app_context().push()
path = '/var/lib/machines/busting/etc/puppetlabs/code/environments/production'
files_gen = find(path, type='f', name='*.pp')
files = [f for f in files_gen]
parse_files(files)
if __name__ == '__main__':
main()
|
