path: root/main.py

#!/usr/bin/env python3

import re
import os
import sys
import subprocess
import yaml
import logging
import logging.config


class PackageFilter(logging.Filter):
    """Logging filter which adds a package field."""

    def __init__(self):
        self.package = False

    def filter(self, record):
        if self.package:
            record.package = self.package
        return True


def get_config_files():
    """Return list of possible configuration files."""
    subpath = ['aur-runner', 'config.yaml']
    conf_files = []
    # TODO better name for this, preferably matching the program name
    if d := os.getenv('OVERRIDE_DIR'):
        conf_files.append(os.path.join(d, *subpath))
    if d := os.getenv('XDG_CONFIG_HOME'):
        conf_files.append(os.path.join(d, *subpath))
    if d := os.getenv('HOME'):
        conf_files.append(os.path.join(d, '.config', *subpath))
    if d := os.getenv('XDG_CONFIG_DIRS'):
        for part in d.split(':'):
            conf_files.append(os.path.join(part, *subpath))
    conf_files.append(os.path.join('/etc/xdg', *subpath))
    return conf_files


default_config_yaml = """
package-list: aur-packages.yaml
source-file: '*virtual*'
path:
  - /usr/local/sbin
  - /usr/local/bin
  - /usr/bin
  - /usr/bin/site_perl
  - /usr/bin/vendor_perl
  - /usr/bin/core_perl
cache-dir: cache
pkgdest-dir: dest
logging:
  version: 1
  formatters:
    detailed:
      class: logging.Formatter
      format: '[%(asctime)s] %(levelname)-8s %(message)s'
      datefmt: '%Y-%m-%dT%H:%M:%S'
  handlers:
    console:
      class: logging.StreamHandler
      formatter: detailed
      level: DEBUG
  root:
    level: DEBUG
    handlers:
      - console
"""


default_config = yaml.safe_load(default_config_yaml)


for conf_file in get_config_files():
    try:
        with open(conf_file) as f:
            user_config = yaml.safe_load(f)
            user_config['source-file'] = conf_file
        break
    except FileNotFoundError:
        pass
else:
    # No config file found
    user_config = {}


def get_conf(key, default=None):
    if default:
        return user_config.get(key, default_config.get(key, default))
    else:
        return user_config.get(key, default_config[key])


logging.config.dictConfig(get_conf('logging'))
logger = logging.getLogger(__name__)
filter = PackageFilter()
logger.addFilter(filter)


log_levels = {
        'DEBUG': logging.DEBUG,
        'INFO': logging.INFO,
        'WARNING': logging.WARNING,
        'WARN': logging.WARNING,
        'ERROR': logging.ERROR,
        'ERR': logging.ERROR,
        'CRITICAL': logging.CRITICAL,
        'FATAL': logging.CRITICAL,
        }


def subprocess_with_log(args, capture_output=False, **kv_args):
    """
    Run a process as by subprocess.run, but also log output which isn't captured by user.

    Note that it is currently unspecified what happens if user tries
    to capture stderr.
    """
    if capture_output:
        kv_args['stdout'] = subprocess.PIPE

    if not capture_output and not kv_args.get('stdout'):
        kv_args['stdout'] = subprocess.PIPE
        kv_args['stderr'] = subprocess.STDOUT
        def get_port(process): return process.stdout
    else:
        kv_args['stderr'] = subprocess.PIPE
        def get_port(process): return process.stderr

    if type(args) == list:
        process_name = args[0]
    elif type(args) == str:
        process_name = args.split(' ')[0]
    else:
        process_name = 'Unknown'

    process = subprocess.Popen(args, **kv_args,)
    logger.info('pid = %i, exec(%s)', process.pid, process.args)

    if kv_args.get('text'):
        sentinel = ''
    else:
        sentinel = b''

    with get_port(process) as pipe:
        for line in iter(pipe.readline, sentinel):
            level = logging.DEBUG
            try:
                if type(line) == bytes:
                    line = line.decode('UTF-8').rstrip()
                # line guaranteed a string here
                if m := re.match('==> ([A-Z]+): (.*)', line):
                    level = log_levels.get(m[1], logging.DEBUG)
                    line = m[2]
            except UnicodeDecodeError:
                pass

            logger.log(level, '%s', line, extra={
                'pid': process.pid,
                'process_name': process_name,
                })

    process.wait()

    if capture_output:
        process.stdout = process.stdout.read()

    if process.returncode == 0:
        logger.info('%i exit success', process.pid)
    else:
        logger.warning('%i exit failure = %s', process.pid, process.returncode)
    return process


auracle_args = ['--color=never']
pacman_args = ['--noconfirm', '--asdeps', '--noprogressbar', '--needed']
makepkg_args = ['--nocolor']


def gather_packages(pkgs):
    """
    Figure out which packages to install, and in which order.

    Takes a list of package names, and returns to lists, one of
    packages which are alreaddy available in the repos, and a list of
    packages which needs to be fetched from the aur. Both are in
    dependency order, and all repo packages should be assumed to be
    required for the aur packages to be build/installed.
    """
    repo_pkgs = []
    aur_pkgs = []
    cmd = subprocess_with_log(
            ['auracle', *auracle_args, 'buildorder', *pkgs],
            capture_output=True, text=True)

    for line in cmd.stdout.split('\n'):
        if not line:
            continue
        pat = r'(SATISFIED|TARGET)?(AUR|REPOS|UNKNOWN) ([^ ]*) ?(.*)'
        m = re.match(pat, line)
        status = m[1]
        source = m[2]
        package = m[3]
        if status == 'SATISFIED':
            logger.debug('Package already installed: %s', package)
            continue
        if source == 'REPOS':
            logger.debug('Would install from repo: %s', package)
            repo_pkgs.append(package)
        elif source == 'AUR':
            logger.debug('Would install from aur: %s', package)
            aur_pkgs.append(package)
        elif source == 'UNKNOWN':
            # auracle buildorder guile-chickadee fails with
            # 'UNKNOWN guile-opengl guile-chickadee'
            # Since guile-opengl is provided by guile-opengl-git, and
            # auracle doesn't find that. The information is however there
            # on the aur: https://aur.archlinux.org/packages/guile-chickadee/
            logger.fatal('AURACLE UNKNOWN, [%s], [%s]', line, m[4])
            sys.exit(1)

    return repo_pkgs, aur_pkgs

##################################################


def main():

    logger.info('Starting run')

    with open(get_conf('package-list')) as f:
        data = yaml.safe_load(f)

    if type(data) != dict:
        logger.fatal('Package must have a dict as root')
        sys.exit(1)

    if type(data.get('packages')) != list:
        logger.fatal('Invalid key packages in package list')
        sys.exit(1)

    pkgs_ = data.get('packages')

    # Flat list of all packages to operate on
    pkg_names = []
    # Mapping from package name to extra args for makepkg
    pkg_options = {}
    # Mapping from package name to extra environment to makepkg
    pkg_envs = {}
    for pkg in pkgs_:
        if type(pkg) == str:
            pkg_names.append(pkg)
        elif type(pkg) == dict:
            pkg_names.append(pkg['name'])
            pkg_options[pkg['name']] = pkg.get('options', '').split(' ')
            pkg_envs[pkg['name']] = pkg.get('env', {})
            for file in pkg.get('files', []):
                with open(file['name'], 'w') as f:
                    f.write(file['content'])
        else:
            logger.fatal('Package entries must be string or dict, not %s',
                         type(pkg))
            sys.exit(1)

    # os.path.join discards earlier components whenever it finds an
    # absolute path
    cachedir = os.path.join(os.getcwd(), get_conf('cache-dir'))
    pkgdest = os.path.join(os.getcwd(), get_conf('pkgdest-dir'))

    for dir in [cachedir, pkgdest]:
        try:
            os.mkdir(dir)
        except FileExistsError:
            pass

    path = get_conf('path')

    repo_pkgs, aur_pkgs = gather_packages(pkg_names)

    if repo_pkgs:
        logger.info('Installing from the repos: %s', repo_pkgs)
    subprocess_with_log(['sudo', 'pacman', *pacman_args, '-Syu', *repo_pkgs])

    failed_packages = []
    for package in aur_pkgs:
        filter.package = package
        args = ['auracle', *auracle_args,
                '--chdir', cachedir,
                'clone', package]
        cmd = subprocess_with_log(args, capture_output=True, text=True)
        m = re.match('[^:]*: (.*)', cmd.stdout)
        if not m:
            logger.error('Auracle clone had unexpected output [%s], '
                         'skipping package [%s]',
                         cmd.stdout, package)
            continue
        cwd = m[1]
        # Allow extra environments, but force our as non-overridable
        env = {**pkg_envs.get(package, {}),
               'PKGDEST': pkgdest,
               'PATH': ':'.join(path),
               }
        extra_opts = pkg_options.get(package, [])
        args = ['makepkg', *makepkg_args, *extra_opts,
                '--install', *pacman_args]
        cmd = subprocess_with_log(args, env=env, cwd=cwd)
        if cmd.returncode != 0:
            failed_packages.append(package)

    filter.package = False

    if failed_packages:
        logger.warning('The following packages failed: %s',
                       failed_packages)
    else:
        logger.info('All packages built successfully')


if __name__ == '__main__':
    if os.getuid() == 0:
        logger.fatal("Running as root isn't supported by makepkg, exiting")
        sys.exit(1)
    main()