1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
#!/usr/bin/env python3
"""
Loads all puppet files in environment, parse them, and store the
parsed data in the database.
"""
import subprocess
import json
import os
import time
import pyenc
from pyenc.db import db
import pyenc.model as model
def find(path, **kvs):
"""Wrapper around find(1)."""
cmdline = ['find', path]
for k, v in kvs.items():
cmdline.append(f'-{k}')
cmdline.append(v)
cmdline.append('-print0')
cmd = subprocess.run(cmdline, capture_output=True)
return (f for f in cmd.stdout.split(b'\0') if f)
class PuppetParseError(Exception):
def __init__(self, code, msg):
self.code = code
self.msg = msg
def __repr__(self):
return f'PuppetParserError({self.code}, {self.msg})'
def __str__(self):
return repr(self)
def puppet_parse(file):
cmd = subprocess.Popen(
['puppet', 'parser', 'dump', '--format', 'json', file],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if cmd.returncode and cmd.returncode != 0:
raise PuppetParseError(cmd.returncode, cmd.stderr.read().decode('UTF-8'))
else:
json = cmd.stdout.read()
if (value := cmd.wait()) != 0:
raise PuppetParseError(value, cmd.stderr.read().decode('UTF-8'))
return json
def parse_files(files):
for i, file in enumerate(files):
try:
st = os.stat(file)
last_modify = st.st_mtime
old_object = model.PuppetFile.query \
.where(model.PuppetFile.path == file) \
.first()
if old_object and old_object.last_parse > last_modify:
# file unchanged since our last parse, skip
continue
print(f'{i}/{len(files)}: {file}')
if old_object:
m = old_object
else:
m = model.PuppetFile(path=file)
m.last_parse = time.time()
m.json = puppet_parse(file)
yield m
except PuppetParseError as e:
# TODO cache error
print('Error:', e)
continue
def interpret_file(json_data):
"""Find all classes in json-representation of file."""
top = json_data['^']
if top[0] == 'class':
tmp = top[1]['#']
idx = tmp.index('name')
return [tmp[idx + 1]]
# print(tmp[idx + 1])
elif top[0] == 'block':
ret_value = []
for element in top[1:]:
if element['^'][0] == 'class':
tmp = element['^'][1]['#']
idx = tmp.index('name')
ret_value.append(tmp[idx + 1])
return ret_value
else:
return []
def main():
app = pyenc.create_app()
app.app_context().push()
path = '/var/lib/machines/busting/etc/puppetlabs/code/environments/production'
files_gen = find(path, type='f', name='*.pp')
files = [f for f in files_gen]
try:
for puppet_file in parse_files(files):
db.session.add(puppet_file)
finally:
db.session.commit()
try:
for puppet_file in model.PuppetFile.query.all():
try:
class_names = interpret_file(json.loads(puppet_file.json))
for class_name in class_names:
db.session.add(model.PuppetClass(
class_name=class_name,
comes_from=puppet_file))
except Exception as e:
print(e)
print(f'Failed: {puppet_file.path}')
finally:
db.session.commit()
if __name__ == '__main__':
main()
|
