Running ansible in the chroot

This is part of a series of posts on the design and technical steps of creating Himblick, a digital signage box based on the Raspberry Pi 4.

In modern times, there are tools for provisioning systems that do useful things and allow to store an entire system configuration in text files committed to git. They are good in being able to reproducibly setup a system, and being able to inspect its contents from looking at the provisioning configuration instead of wading into it.

I normally use Ansible. It does have a chroot connector, but it has some serious limitations.

The biggest issue is that ansible's chroot connector does not mount /dev, /proc and so on, which greatly limits what can be run inside it. Specifically, installing many .deb packages will fail.

We work around it by copying Ansible needs inside the chroot (including Ansible itself), and then run it under systemd-nspawn using the local connector.

Systemd ansible operations still don't work despite upstream having closed the issue, so playbooks cannot use the systemd module.

Systemd is however able to perform operations inside the chroot using the --root option, so we can do enabling and disabling in himblick and leave the rest of the provisioning to Ansible:

    def systemctl_enable(self, unit: str):
        """
        Enable (and if needed unmask) the given systemd unit
        """
        with self.working_resolvconf("/etc/resolv.conf"):
            env = dict(os.environ)
            env["LANG"] = "C"
            subprocess.run(["systemctl", "--root=" + self.root, "enable", unit], check=True, env=env)
            subprocess.run(["systemctl", "--root=" + self.root, "unmask", unit], check=True, env=env)

    def systemctl_disable(self, unit: str, mask=True):
        """
        Disable (and optionally mask) the given systemd unit
        """
        with self.working_resolvconf("/etc/resolv.conf"):
            env = dict(os.environ)
            env["LANG"] = "C"
            subprocess.run(["systemctl", "--root=" + self.root, "disable", unit], check=True, env=env)
            if mask:
                subprocess.run(["systemctl", "--root=" + self.root, "mask", unit], check=True, env=env)

This is the code that runs Ansible in the chroot. Leaving the playbook inside /srv/himblick makes it possible to try tweaks in the running system during development:

    def run_ansible(self, playbook, roles, host_vars):
        """
        Run ansible inside the chroot
        """
        # Make sure ansible is installed in the chroot
        self.apt_install("ansible")

        # Create an ansible environment inside the rootfs
        ansible_dir = self.abspath("/srv/himblick/ansible", create=True)

        # Copy the ansible playbook and roles
        self.copy_to("rootfs.yaml", "/srv/himblick/ansible")
        self.copy_to("roles", "/srv/himblick/ansible")

        # Write the variables
        vars_file = os.path.join(ansible_dir, "himblick-vars.yaml")
        with open(vars_file, "wt") as fd:
            yaml.dump(host_vars, fd)

        # Write ansible's inventory
        ansible_inventory = os.path.join(ansible_dir, "inventory.ini")
        with open(ansible_inventory, "wt") as fd:
            print("[rootfs]", file=fd)
            print("localhost ansible_connection=local", file=fd)

        # Write ansible's config
        ansible_cfg = os.path.join(ansible_dir, "ansible.cfg")
        with open(ansible_cfg, "wt") as fd:
            print("[defaults]", file=fd)
            print("nocows = 1", file=fd)
            print("inventory = inventory.ini", file=fd)
            print("[inventory]", file=fd)
            # See https://github.com/ansible/ansible/issues/48859
            print("enable_plugins = ini", file=fd)

        # Write ansible's startup script
        args = ["exec", "ansible-playbook", "-v", "rootfs.yaml"]
        ansible_sh = os.path.join(ansible_dir, "rootfs.sh")
        with open(ansible_sh, "wt") as fd:
            print("#!/bin/sh", file=fd)
            print("set -xue", file=fd)
            print('cd $(dirname -- "$0")', file=fd)
            print("export ANSIBLE_CONFIG=ansible.cfg", file=fd)
            print(" ".join(shlex.quote(x) for x in args), file=fd)
        os.chmod(ansible_sh, 0o755)

        # Run ansible in the chroot using systemd-nspawn
        self.run(["/srv/himblick/ansible/rootfs.sh"], check=True)

    def run(self, cmd: List[str], check=True, **kw) -> subprocess.CompletedProcess:
        """
        Run the given command inside the chroot
        """
        log.info("%s: running %s", self.root, " ".join(shlex.quote(x) for x in cmd))
        chroot_cmd = ["systemd-nspawn", "-D", self.root]
        chroot_cmd.extend(cmd)
        if "env" not in kw:
            kw["env"] = dict(os.environ)
            kw["env"]["LANG"] = "C"
        with self.working_resolvconf("/etc/resolv.conf"):
            return subprocess.run(chroot_cmd, check=check, **kw)

Finally, to speed things up, here's a trick to cache the .deb files downloaded during provisioning, and reuse them for the following runs:

    def save_apt_cache(self, chroot: Chroot):
        """
        Copy .deb files from the apt cache in the rootfs to our local cache
        """
        if not self.cache:
            return

        rootfs_apt_cache = chroot.abspath("/var/cache/apt/archives")
        apt_cache_root = self.cache.get("apt")

        for fn in os.listdir(rootfs_apt_cache):
            if not fn.endswith(".deb"):
                continue
            src = os.path.join(rootfs_apt_cache, fn)
            dest = os.path.join(apt_cache_root, fn)
            if os.path.exists(dest) and os.path.getsize(dest) == os.path.getsize(src):
                continue
            shutil.copy(src, dest)

    def restore_apt_cache(self, chroot: Chroot):
        """
        Copy .deb files from our local cache to the apt cache in the rootfs
        """
        if not self.cache:
            return

        rootfs_apt_cache = chroot.abspath("/var/cache/apt/archives")
        apt_cache_root = self.cache.get("apt")

        for fn in os.listdir(apt_cache_root):
            if not fn.endswith(".deb"):
                continue
            src = os.path.join(apt_cache_root, fn)
            dest = os.path.join(rootfs_apt_cache, fn)
            if os.path.exists(dest) and os.path.getsize(dest) == os.path.getsize(src):
                continue
            shutil.copy(src, dest)