diff options
Diffstat (limited to 'pyenc/app')
-rw-r--r-- | pyenc/app/__init__.py | 0 | ||||
-rw-r--r-- | pyenc/app/api.py | 117 | ||||
-rw-r--r-- | pyenc/app/cmdline.py | 37 | ||||
-rw-r--r-- | pyenc/app/model.py | 191 |
4 files changed, 345 insertions, 0 deletions
diff --git a/pyenc/app/__init__.py b/pyenc/app/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pyenc/app/__init__.py diff --git a/pyenc/app/api.py b/pyenc/app/api.py new file mode 100644 index 0000000..a8a37d9 --- /dev/null +++ b/pyenc/app/api.py @@ -0,0 +1,117 @@ +import flask +from flask import ( + Blueprint, + request, + url_for +) + +from . import model + + +def concatenate(listlist): + result = [] + for lst in listlist: + result.extend(lst) + return result + + +api = Blueprint('api', __name__) + + +@api.route('/classes') +def list_classes(): + """Return all classes fuzzy matching q.""" + environment = request.args.get('environment', 'production') + query = request.args.get('q', '') + wildcarded_query = '%{}%'.format('%'.join(query.split(' '))) + + result = model.db.engine.execute(model.db.text(""" + SELECT pc.class_name + FROM environment_classes ec + LEFT JOIN puppet_environment e ON ec.environment_id = e.id + LEFT JOIN puppet_class pc ON ec.class_id = pc.id + WHERE e.name = :environment + AND pc.class_name LIKE :wildcard + """), { + 'environment': environment, + 'wildcard': wildcarded_query, + }) + + return flask.json.jsonify([x for (x,) in result]) + + # results = \ + # model \ + # .PuppetClass \ + # .query \ + # .where(model.PuppetClass.class_name.like(wildcarded_query)) \ + # .where(model.PuppetClass.environments.name == environment) \ + # .all() + # print(wildcarded_query) + # return flask.json.jsonify([x.class_name for x in results]) + + +@api.route('/environments') +def list_environments(): + envs = model.PuppetEnvironment.query.all() + return flask.json.jsonify([env.name for env in envs]) + + +@api.route('/class-file') +def class_file(): + class_name = request.args.get('class') + result = model.PuppetClass.query \ + .where(model.PuppetClass.class_name == class_name) \ + .all() + return flask.json.jsonify(concatenate([ + [f.path for f in x.files] + for x in result])) + + +@api.route('/hosts') +def hosts(): + result = model.Host.query.all() + return flask.json.jsonify([x.fqdn for x in result]) + + +@api.route('/classes-for') +def classes_for(): + """Return classes mapped to host `fqdn'.""" + fqdn = request.args.get('fqdn') + classes = [cls.class_name + for cls in model.Host.query.where(model.Host.fqdn == fqdn) + .first().classes] + return flask.json.jsonify(classes) + + +@api.route('/change-classes', methods=['POST']) +def change_classes(): + """ + Update set of active classes for node. + + Takes a json object as the POST body, which should have the keys + - fqdn :: which host to operate on + - removed :: classes which should be removed + - added :: classes which should be added + """ + j = request.json + host = model.Host.query.where(model.Host.fqdn == j['fqdn']).first() + remove_set = set(j['removed']) + + new_cls = [] + for cls in host.classes: + if cls.class_name in remove_set: + continue + new_cls.append(cls) + host.classes = new_cls + + cls = model.PuppetClass.query \ + .where(model.PuppetClass.class_name.in_(j['added'])) \ + .all() + host.classes.extend(cls) + # print(remove_set, db.db.session.dirty) + return flask.redirect(url_for('classes_for', fqdn=j['fqdn'])) + + +def init_app(app): + """Register blueprint to app.""" + app.register_blueprint(api, url_prefix='/api') diff --git a/pyenc/app/cmdline.py b/pyenc/app/cmdline.py new file mode 100644 index 0000000..4e7e33f --- /dev/null +++ b/pyenc/app/cmdline.py @@ -0,0 +1,37 @@ +import click +from flask.cli import AppGroup + +app_group = AppGroup('user', help="Testt") + + +@app_group.command('enc') +@click.argument('fqdn') +def enc(fqdn): + from pyenc import enc + enc.run_enc(fqdn) + + +@app_group.command('init-db') +def initialize_database(): + from pyenc.app import model + model.db.create_all() + # model.db.session.add(model.Misc(key='db-version', value='0.1')) + # model.db.session.add(model.Misc(key='default-table', value='production')) + # model.db.session.commit() + + +@app_group.command('enumerate-classes') +@click.argument('environment') +def enumerate_classes(environment): + from pyenc import enumerate_classes as enumerate_classes_ + environment_name = environment + # TODO path should be an argument + path_base = '/var/lib/machines/busting/etc/puppetlabs/code/environments/' + enumerate_classes_.run( + path_base=path_base, + environment_name=environment_name) + + +def init_app(app): + """Add command line options to current flask app.""" + app.cli.add_command(app_group) diff --git a/pyenc/app/model.py b/pyenc/app/model.py new file mode 100644 index 0000000..f67cd10 --- /dev/null +++ b/pyenc/app/model.py @@ -0,0 +1,191 @@ +"""Database model for application.""" + +from flask_sqlalchemy import SQLAlchemy + +# db = SQLAlchemy(session_options={"autoflush": False}) +db = SQLAlchemy() + + +def init_app(app): + """Adds database bindings to a Flask App.""" + db.init_app(app) + import logging + # logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + + +host_classes = db.Table( + 'host_classes', + db.Column('host_id', db.ForeignKey('host.id'), primary_key=True), + db.Column('class_id', db.ForeignKey('puppet_class.id'), primary_key=True), +) + + +# NOTE this is non-final, and might get removed shortly +environment_classes = db.Table( + 'environment_classes', + db.Column('environment_id', db.ForeignKey('puppet_environment.id'), primary_key=True), + db.Column('class_id', db.ForeignKey('puppet_class.id'), primary_key=True), +) + + +class_files = db.Table( + 'class_files', + db.Column('class_id', db.ForeignKey('puppet_class.id'), primary_key=True), + db.Column('file_id', db.ForeignKey('puppet_file.id'), primary_key=True), + db.UniqueConstraint('class_id', 'file_id'), +) + + +# class HostClasses(db.Model): +# __tablename__ = 'host_classes' +# id = db.Column(db.Integer, primary_key=True) +# host_id = db.Column(db.Integer, db.ForeignKey('host.id'), nullable=False) +# class_id = db.Column(db.Integer, db.ForeignKey('puppet_class.id'), nullable=False) + + +class PuppetEnvironment(db.Model): + """ + A puppet environment. + + An enviromnet is a collection of modules, but here we only keep + the files of the modules, in PuppetFile. + """ + __tablename__ = 'puppet_environment' + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.Text, nullable=False, unique=True) + classes = db.relationship( + 'PuppetClass', + back_populates='environments', + secondary=environment_classes) + hosts = db.relationship( + 'Host', + back_populates='environment') + + +class Host(db.Model): + """ + Single computer. + + A computer has a name (machine.example.com.), an environment + (production) and a list of puppet classes. + + (TODO and direct values?) + """ + + __tablename__ = 'host' + id = db.Column(db.Integer, primary_key=True) + fqdn = db.Column(db.Text, nullable=False, unique=True) + environment_id = db.Column(db.Integer, db.ForeignKey(f'{PuppetEnvironment.__tablename__}.id')) + environment = db.relationship('PuppetEnvironment', back_populates='hosts') + # classes = db.relationship('HostClasses', backref='host', lazy='dynamic') + classes = db.relationship( + 'PuppetClass', + back_populates='hosts', + secondary=host_classes) + + def serialize(self): # pylint: disable=missing-function-docstring + return {column.name: self.__getattribute__(column.name) + for column in self.__table__.columns} + + +class PuppetFile(db.Model): + """ + Puppet source code file. + + Keeps track of known puppet files. Each file contains 0 to many + puppet classes. + + Each file is uniquely identified by the pair (path, environment). + """ + + __tablename__ = 'puppet_file' + id = db.Column(db.Integer, primary_key=True) + # Where we found the file (path inside environment) + # e.g. /etc/puppetlabs/code/environments/<environment name>/<this path> + path = db.Column(db.Text, nullable=False) + + # Puppet environment this file belongs in + environment = db.Column(db.Integer, + db.ForeignKey(f'{PuppetEnvironment.__tablename__}.id'), + nullable=False) + + # Checksum of the content, should be usable as a key in PuppetFileContent + checksum = db.Column(db.Text, nullable=False) + + # When we last read data into json + last_parse = db.Column(db.Float) + + classes = db.relationship('PuppetClass', + back_populates='files', + secondary=class_files) + content = db.relationship('PuppetFileContent', backref='file') + + __table_args__ = ( + db.UniqueConstraint('path', 'environment'), + ) + + +class PuppetFileContent(db.Model): + """ + (Parsed) contents of puppet source files. + + Separate from PuppetFile since many environments can share files, + and I don't want to store reduntand data. + """ + __tablename__ = 'puppet_file_content' + + id = db.Column(db.Integer, primary_key=True) + + file_id = db.Column(db.Integer, db.ForeignKey(f'{PuppetFile.__tablename__}.id')) + + # Checksum of the original file + checksum = db.Column(db.Text, nullable=False) + + # Output of 'puppet parser dump --format json <filename>' + json = db.Column(db.Text, nullable=False) + + +# TODO class environment mappings? +# - the same class can exist in multiple environmentns +# - the same class in multiple environments might be different +# - the class can come or go when the file is changed (??) +# - when a node changes environment it still has its classes, but they +# refer to something else now + +# Possibly: +# nodes holds its list of classes as a list of strings +# I have tables which maps class names to files per environment + +# What happens when two different environments have separate classes +# which share a name + +class PuppetClass(db.Model): + """ + A puppet class. + + The class itself only keeps track of its name here, and mostly + ensures that only existing classes can be added to a given node/host. + """ + __tablename__ = 'puppet_class' + id = db.Column(db.Integer, primary_key=True) + class_name = db.Column(db.Text, nullable=False, unique=True) + + hosts = db.relationship( + 'Host', + back_populates='classes', + secondary=host_classes) + environments = db.relationship( + 'PuppetEnvironment', + back_populates='classes', + secondary=environment_classes) + files = db.relationship( + 'PuppetFile', + back_populates='classes', + secondary=class_files) + + +class Misc(db.Model): + __tablename__ = 'misc' + id = db.Column(db.Integer, primary_key=True) + key = db.Column(db.Text, nullable=False) + value = db.Column(db.Text) |