From 402a5afdb5fc9b73daa3cd33400ede300d26b793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20H=C3=B6rnquist?= Date: Mon, 8 Aug 2022 20:36:37 +0200 Subject: Rename some database stuff. --- import_yaml.py | 6 ++--- pyenc/app/api.py | 8 +++---- pyenc/app/model.py | 59 +++++++++++++++++----------------------------- pyenc/enumerate_classes.py | 22 ++++++++--------- 4 files changed, 39 insertions(+), 56 deletions(-) diff --git a/import_yaml.py b/import_yaml.py index d602cad..0007e75 100755 --- a/import_yaml.py +++ b/import_yaml.py @@ -19,16 +19,16 @@ for fqdn, val in data.items(): h = model.Host.query.where(model.Host.fqdn == fqdn).first() if not h: h = model.Host(fqdn=fqdn) - h.environment = model.PuppetEnvironment \ + h.environment = model.Environment \ .query \ - .where(model.PuppetEnvironment.name == val.get('environment', 'production')) \ + .where(model.Environment.name == val.get('environment', 'production')) \ .one() classes = val['classes'] if type(classes) == dict: classes = classes.keys() cls = model.PuppetClass.query \ - .where(model.PuppetClass.class_name.in_(classes)).all() + .where(model.PuppetClass.name.in_(classes)).all() print(cls) h.classes.extend(cls) diff --git a/pyenc/app/api.py b/pyenc/app/api.py index a8a37d9..cb80da8 100644 --- a/pyenc/app/api.py +++ b/pyenc/app/api.py @@ -43,7 +43,7 @@ def list_classes(): # model \ # .PuppetClass \ # .query \ - # .where(model.PuppetClass.class_name.like(wildcarded_query)) \ + # .where(model.PuppetClass.name.like(wildcarded_query)) \ # .where(model.PuppetClass.environments.name == environment) \ # .all() # print(wildcarded_query) @@ -52,7 +52,7 @@ def list_classes(): @api.route('/environments') def list_environments(): - envs = model.PuppetEnvironment.query.all() + envs = model.Environment.query.all() return flask.json.jsonify([env.name for env in envs]) @@ -60,7 +60,7 @@ def list_environments(): def class_file(): class_name = request.args.get('class') result = model.PuppetClass.query \ - .where(model.PuppetClass.class_name == class_name) \ + .where(model.PuppetClass.name == class_name) \ .all() return flask.json.jsonify(concatenate([ [f.path for f in x.files] @@ -105,7 +105,7 @@ def change_classes(): host.classes = new_cls cls = model.PuppetClass.query \ - .where(model.PuppetClass.class_name.in_(j['added'])) \ + .where(model.PuppetClass.name.in_(j['added'])) \ .all() host.classes.extend(cls) # print(remove_set, db.db.session.dirty) diff --git a/pyenc/app/model.py b/pyenc/app/model.py index f67cd10..c37cec3 100644 --- a/pyenc/app/model.py +++ b/pyenc/app/model.py @@ -23,7 +23,7 @@ host_classes = db.Table( # NOTE this is non-final, and might get removed shortly environment_classes = db.Table( 'environment_classes', - db.Column('environment_id', db.ForeignKey('puppet_environment.id'), primary_key=True), + db.Column('environment_id', db.ForeignKey('environment.id'), primary_key=True), db.Column('class_id', db.ForeignKey('puppet_class.id'), primary_key=True), ) @@ -32,25 +32,17 @@ class_files = db.Table( 'class_files', db.Column('class_id', db.ForeignKey('puppet_class.id'), primary_key=True), db.Column('file_id', db.ForeignKey('puppet_file.id'), primary_key=True), - db.UniqueConstraint('class_id', 'file_id'), ) -# class HostClasses(db.Model): -# __tablename__ = 'host_classes' -# id = db.Column(db.Integer, primary_key=True) -# host_id = db.Column(db.Integer, db.ForeignKey('host.id'), nullable=False) -# class_id = db.Column(db.Integer, db.ForeignKey('puppet_class.id'), nullable=False) - - -class PuppetEnvironment(db.Model): +class Environment(db.Model): """ A puppet environment. An enviromnet is a collection of modules, but here we only keep the files of the modules, in PuppetFile. """ - __tablename__ = 'puppet_environment' + __tablename__ = 'environment' id = db.Column(db.Integer, primary_key=True) name = db.Column(db.Text, nullable=False, unique=True) classes = db.relationship( @@ -60,6 +52,9 @@ class PuppetEnvironment(db.Model): hosts = db.relationship( 'Host', back_populates='environment') + files = db.relationship( + 'PuppetFile', + back_populates='environment') class Host(db.Model): @@ -75,9 +70,10 @@ class Host(db.Model): __tablename__ = 'host' id = db.Column(db.Integer, primary_key=True) fqdn = db.Column(db.Text, nullable=False, unique=True) - environment_id = db.Column(db.Integer, db.ForeignKey(f'{PuppetEnvironment.__tablename__}.id')) - environment = db.relationship('PuppetEnvironment', back_populates='hosts') - # classes = db.relationship('HostClasses', backref='host', lazy='dynamic') + + environment_id = db.Column(db.Integer, db.ForeignKey(f'{Environment.__tablename__}.id')) + environment = db.relationship('Environment', back_populates='hosts') + classes = db.relationship( 'PuppetClass', back_populates='hosts', @@ -95,19 +91,20 @@ class PuppetFile(db.Model): Keeps track of known puppet files. Each file contains 0 to many puppet classes. - Each file is uniquely identified by the pair (path, environment). + Each file is uniquely identified by the pair (path, environment_id). """ __tablename__ = 'puppet_file' id = db.Column(db.Integer, primary_key=True) - # Where we found the file (path inside environment) - # e.g. /etc/puppetlabs/code/environments// + # Where we found the file (path inside environment_id) + # e.g. /etc/puppetlabs/code/environments// path = db.Column(db.Text, nullable=False) - # Puppet environment this file belongs in - environment = db.Column(db.Integer, - db.ForeignKey(f'{PuppetEnvironment.__tablename__}.id'), - nullable=False) + # Puppet environment_id this file belongs in + environment_id = db.Column(db.Integer, + db.ForeignKey(f'{Environment.__tablename__}.id'), + nullable=False) + environment = db.relationship('Environment', back_populates='files') # Checksum of the content, should be usable as a key in PuppetFileContent checksum = db.Column(db.Text, nullable=False) @@ -121,7 +118,7 @@ class PuppetFile(db.Model): content = db.relationship('PuppetFileContent', backref='file') __table_args__ = ( - db.UniqueConstraint('path', 'environment'), + db.UniqueConstraint('path', 'environment_id'), ) @@ -145,20 +142,6 @@ class PuppetFileContent(db.Model): json = db.Column(db.Text, nullable=False) -# TODO class environment mappings? -# - the same class can exist in multiple environmentns -# - the same class in multiple environments might be different -# - the class can come or go when the file is changed (??) -# - when a node changes environment it still has its classes, but they -# refer to something else now - -# Possibly: -# nodes holds its list of classes as a list of strings -# I have tables which maps class names to files per environment - -# What happens when two different environments have separate classes -# which share a name - class PuppetClass(db.Model): """ A puppet class. @@ -168,14 +151,14 @@ class PuppetClass(db.Model): """ __tablename__ = 'puppet_class' id = db.Column(db.Integer, primary_key=True) - class_name = db.Column(db.Text, nullable=False, unique=True) + name = db.Column(db.Text, nullable=False, unique=True) hosts = db.relationship( 'Host', back_populates='classes', secondary=host_classes) environments = db.relationship( - 'PuppetEnvironment', + 'Environment', back_populates='classes', secondary=environment_classes) files = db.relationship( diff --git a/pyenc/enumerate_classes.py b/pyenc/enumerate_classes.py index d4197bd..c6ccf47 100644 --- a/pyenc/enumerate_classes.py +++ b/pyenc/enumerate_classes.py @@ -122,9 +122,9 @@ def enumerate_files(path_base, environment): # This works in at least postgres and sqlite3 db.engine.execute(text(""" - INSERT INTO puppet_file (path, environment, checksum) + INSERT INTO puppet_file (path, environment_id, checksum) VALUES (:path, :environment, :checksum) - ON CONFLICT (path, environment) + ON CONFLICT (path, environment_id) DO UPDATE SET checksum = EXCLUDED.checksum """), { 'path': os.path.relpath(puppet_file.path, path), @@ -144,13 +144,13 @@ def ensure_environment(name): otherwise create it and return that. """ environment = model \ - .PuppetEnvironment \ + .Environment \ .query \ - .where(model.PuppetEnvironment.name == name) \ + .where(model.Environment.name == name) \ .one_or_none() if not environment: - environment = model.PuppetEnvironment(name=name) + environment = model.Environment(name=name) db.session.add(environment) # This also updates our environment object, filling in # autogenerated fieldsfields. @@ -187,8 +187,8 @@ def run(path_base: Path = '/etc/puppetlabs/code/environments', count = base.count() result = base \ - .join(model.PuppetEnvironment) \ - .add_column(model.PuppetEnvironment.name) \ + .join(model.Environment) \ + .add_column(model.Environment.name) \ .all() db.session.commit() @@ -245,22 +245,22 @@ def run(path_base: Path = '/etc/puppetlabs/code/environments', # Add classs (if not exists) db.engine.execute(text(""" - INSERT INTO puppet_class (class_name) + INSERT INTO puppet_class (name) VALUES (:name) - ON CONFLICT (class_name) DO NOTHING + ON CONFLICT (name) DO NOTHING """), {'name': class_name}) # Add class to environment (if not already there) db.engine.execute(text(""" INSERT INTO environment_classes (environment_id, class_id) - SELECT :env, id FROM puppet_class WHERE class_name = :name + SELECT :env, id FROM puppet_class WHERE puppet_class.name = :name ON CONFLICT (environment_id, class_id) DO NOTHING """), {'env': environment.id, 'name': class_name}) # Add class to file mapping (if not already there) db.engine.execute(text(""" INSERT INTO class_files (file_id, class_id) - SELECT :file, id FROM puppet_class WHERE class_name = :name + SELECT :file, id FROM puppet_class WHERE puppet_class.name = :name ON CONFLICT (file_id, class_id) DO NOTHING """), {'file': file_content.file_id, 'name': class_name}) -- cgit v1.2.3