path: root/nixos/modules/profiles/nix-builder.nix

/*
  Backend-neutral half of the profile that uses NixOS to create a remote
  builder VM to build Linux packages, which can be used to build packages for
  Linux on other operating systems; primarily macOS.

  It contains the relevant guest settings as well as an installer script that
  manages the VM on the host, but no VM backend. A backend module provides
  `config.system.build.vm`, which the installer script launches; see
  ./nix-builder-vm.nix for the QEMU-based composition.
*/
{
  config,
  lib,
  options,
  ...
}:

let
  keysDirectory = "/var/keys";

  user = "builder";

  keyType = "ed25519";

  cfg = config.virtualisation.darwin-builder;

in

{
  imports = [
    # Avoid a dependency on stateVersion
    {
      disabledModules = [
        ../virtualisation/nixos-containers.nix
        ../services/x11/desktop-managers/xterm.nix
      ];
      # swraid's default depends on stateVersion
      config.boot.swraid.enable = false;
      options.boot.isContainer = lib.mkOption {
        default = false;
        internal = true;
      };
      options.boot.isNspawnContainer = lib.mkOption {
        default = false;
        internal = true;
      };
    }
  ];

  options.virtualisation.darwin-builder = with lib; {
    diskSize = mkOption {
      default = 20 * 1024;
      type = types.int;
      example = 30720;
      description = "The maximum disk space allocated to the runner in MiB (1024×1024 bytes).";
    };
    memorySize = mkOption {
      default = 3 * 1024;
      type = types.int;
      example = 8192;
      description = "The runner's memory in MiB (1024×1024 bytes).";
    };
    min-free = mkOption {
      default = 1024 * 1024 * 1024;
      type = types.int;
      example = 1073741824;
      description = ''
        The threshold (in bytes) of free disk space left at which to
        start garbage collection on the runner
      '';
    };
    max-free = mkOption {
      default = 3 * 1024 * 1024 * 1024;
      type = types.int;
      example = 3221225472;
      description = ''
        The threshold (in bytes) of free disk space left at which to
        stop garbage collection on the runner
      '';
    };
    workingDirectory = mkOption {
      default = ".";
      type = types.str;
      example = "/var/lib/darwin-builder";
      description = ''
        The working directory to use to run the script. When running
        as part of a flake will need to be set to a non read-only filesystem.
      '';
    };
    hostPort = mkOption {
      default = 31022;
      type = types.port;
      example = 22;
      description = ''
        The localhost host port to forward TCP to the guest port.
      '';
    };
  };

  config = {
    # The builder is not intended to be used interactively
    documentation.enable = false;

    environment.etc = {
      "ssh/ssh_host_ed25519_key" = {
        mode = "0600";

        source = ./keys/ssh_host_ed25519_key;
      };

      "ssh/ssh_host_ed25519_key.pub" = {
        mode = "0644";

        source = ./keys/ssh_host_ed25519_key.pub;
      };
    };

    # The linux builder is a lightweight VM for remote building; not evaluation.
    nix.channel.enable = false;

    # Deployment is by image.
    # TODO system.switch.enable = false;?
    system.disableInstallerTools = true;

    # Allow the system derivation to be substituted, so that
    # users are less likely to run into a state where they need
    # the builder running to build the builder if they just want
    # to make a tweak that only affects the macOS side of things,
    # like changing the QEMU args.
    #
    # TODO(winter): Move to qemu-vm? Trying it here for now as a
    # low impact change that'll probably improve people's experience.
    system.systemBuilderArgs.allowSubstitutes = true;

    nix.settings = {
      min-free = cfg.min-free;

      max-free = cfg.max-free;

      trusted-users = [ user ];
    };

    services = {
      getty.autologinUser = user;

      openssh = {
        enable = true;

        authorizedKeysFiles = [ "${keysDirectory}/%u_${keyType}.pub" ];
      };
    };

    system.build.macos-builder-installer =
      let
        privateKey = "/etc/nix/${user}_${keyType}";

        publicKey = "${privateKey}.pub";

        # This installCredentials script is written so that it's as easy as
        # possible for a user to audit before confirming the `sudo`
        installCredentials = hostPkgs.writeShellScript "install-credentials" ''
          set -euo pipefail

          KEYS="''${1}"
          INSTALL=${hostPkgs.coreutils}/bin/install
          "''${INSTALL}" -g nixbld -m 600 "''${KEYS}/${user}_${keyType}" ${privateKey}
          "''${INSTALL}" -g nixbld -m 644 "''${KEYS}/${user}_${keyType}.pub" ${publicKey}
        '';

        hostPkgs = config.virtualisation.host.pkgs;

        add-keys = hostPkgs.writeShellScriptBin "add-keys" (
          ''
            set -euo pipefail
          ''
          +
            # When running as non-interactively as part of a DarwinConfiguration the working directory
            # must be set to a writeable directory.
            (
              if cfg.workingDirectory != "." then
                ''
                  ${hostPkgs.coreutils}/bin/mkdir --parent "${cfg.workingDirectory}"
                  cd "${cfg.workingDirectory}"
                ''
              else
                ""
            )
          + ''
            KEYS="''${KEYS:-./keys}"
            ${hostPkgs.coreutils}/bin/mkdir --parent "''${KEYS}"
            PRIVATE_KEY="''${KEYS}/${user}_${keyType}"
            PUBLIC_KEY="''${PRIVATE_KEY}.pub"
            if [ ! -e "''${PRIVATE_KEY}" ] || [ ! -e "''${PUBLIC_KEY}" ]; then
                ${hostPkgs.coreutils}/bin/rm --force -- "''${PRIVATE_KEY}" "''${PUBLIC_KEY}"
                ${hostPkgs.openssh}/bin/ssh-keygen -q -f "''${PRIVATE_KEY}" -t ${keyType} -N "" -C 'builder@localhost'
            fi
            if ! ${hostPkgs.diffutils}/bin/cmp "''${PUBLIC_KEY}" ${publicKey}; then
              (set -x; sudo --reset-timestamp ${installCredentials} "''${KEYS}")
            fi
          ''
        );

        run-builder = hostPkgs.writeShellScriptBin "run-builder" ''
          set -euo pipefail
          KEYS="''${KEYS:-./keys}"
          KEYS="$(${hostPkgs.nix}/bin/nix-store --add "$KEYS")" ${lib.getExe config.system.build.vm}
        '';

      in
      hostPkgs.writeTextFile {
        name = "create-builder";
        executable = true;
        destination = "/bin/create-builder";
        text = ''
          #!${hostPkgs.runtimeShell}
          set -euo pipefail
          export KEYS="''${KEYS:-./keys}"
          ${lib.getExe add-keys}
          ${lib.getExe run-builder}
        '';
        checkPhase = ''
          ${hostPkgs.stdenv.shellDryRun} "$target"
        '';
        meta = {
          mainProgram = "create-builder";
          description = "Create a Linux builder VM for macOS";
          platforms = lib.platforms.darwin;
          maintainers = with lib.maintainers; [ booxter ];
        };
        passthru = {
          # Let users in the repl inspect the config
          nixosConfig = config;
          nixosOptions = options;

          inherit add-keys run-builder;
        };
      };

    system = {
      # To prevent gratuitous rebuilds on each change to Nixpkgs
      nixos.revision = null;

      # To prevent channels and Git checkouts resulting in different system drvs
      nixos.versionSuffix = "";

      # to be updated by module maintainers, see nixpkgs#325610
      stateVersion = "24.05";
    };

    users.users."${user}" = {
      isNormalUser = true;
    };

    security.polkit.enable = true;

    security.polkit.extraConfig = ''
      polkit.addRule(function(action, subject) {
        if (action.id === "org.freedesktop.login1.power-off" && subject.user === "${user}") {
          return "yes";
        } else {
          return "no";
        }
      })
    '';

    virtualisation = {
      diskSize = cfg.diskSize;

      memorySize = cfg.memorySize;

      sharedDirectories.keys = {
        source = "\"$KEYS\"";
        target = keysDirectory;
      };

      # Obviously the /nix/store needs to be writable on the guest in order for it
      # to perform builds.
      writableStore = true;

      # This ensures that anything built on the guest isn't lost when the guest is
      # restarted.
      writableStoreUseTmpfs = false;

      # Pass certificates from host to the guest otherwise when custom CA certificates
      # are required we can't use the cached builder.
      useHostCerts = true;
    };
  };
}