diff options
Diffstat (limited to 'nixpkgs/nixos/lib')
52 files changed, 7407 insertions, 0 deletions
diff --git a/nixpkgs/nixos/lib/default.nix b/nixpkgs/nixos/lib/default.nix new file mode 100644 index 000000000000..65d91342d4d1 --- /dev/null +++ b/nixpkgs/nixos/lib/default.nix @@ -0,0 +1,41 @@ +let + # The warning is in a top-level let binding so it is only printed once. + minimalModulesWarning = warn "lib.nixos.evalModules is experimental and subject to change. See nixos/lib/default.nix" null; + inherit (nonExtendedLib) warn; + nonExtendedLib = import ../../lib; +in +{ # Optional. Allows an extended `lib` to be used instead of the regular Nixpkgs lib. + lib ? nonExtendedLib, + + # Feature flags allow you to opt in to unfinished code. These may change some + # behavior or disable warnings. + featureFlags ? {}, + + # This file itself is rather new, so we accept unknown parameters to be forward + # compatible. This is generally not recommended, because typos go undetected. + ... +}: +let + seqIf = cond: if cond then builtins.seq else a: b: b; + # If cond, force `a` before returning any attr + seqAttrsIf = cond: a: lib.mapAttrs (_: v: seqIf cond a v); + + eval-config-minimal = import ./eval-config-minimal.nix { inherit lib; }; + + testing-lib = import ./testing/default.nix { inherit lib; }; +in +/* + This attribute set appears as lib.nixos in the flake, or can be imported + using a binding like `nixosLib = import (nixpkgs + "/nixos/lib") { }`. +*/ +{ + inherit (seqAttrsIf (!featureFlags?minimalModules) minimalModulesWarning eval-config-minimal) + evalModules + ; + + inherit (testing-lib) + evalTest + runTest + ; + +} diff --git a/nixpkgs/nixos/lib/eval-cacheable-options.nix b/nixpkgs/nixos/lib/eval-cacheable-options.nix new file mode 100644 index 000000000000..d26967ebe09b --- /dev/null +++ b/nixpkgs/nixos/lib/eval-cacheable-options.nix @@ -0,0 +1,54 @@ +{ libPath +, pkgsLibPath +, nixosPath +, modules +, stateVersion +, release +}: + +let + lib = import libPath; + modulesPath = "${nixosPath}/modules"; + # dummy pkgs set that contains no packages, only `pkgs.lib` from the full set. + # not having `pkgs.lib` causes all users of `pkgs.formats` to fail. + pkgs = import pkgsLibPath { + inherit lib; + pkgs = null; + }; + utils = import "${nixosPath}/lib/utils.nix" { + inherit config lib; + pkgs = null; + }; + # this is used both as a module and as specialArgs. + # as a module it sets the _module special values, as specialArgs it makes `config` + # unusable. this causes documentation attributes depending on `config` to fail. + config = { + _module.check = false; + _module.args = {}; + system.stateVersion = stateVersion; + }; + eval = lib.evalModules { + modules = (map (m: "${modulesPath}/${m}") modules) ++ [ + config + ]; + specialArgs = { + inherit config pkgs utils; + class = "nixos"; + }; + }; + docs = import "${nixosPath}/doc/manual" { + pkgs = pkgs // { + inherit lib; + # duplicate of the declaration in all-packages.nix + buildPackages.nixosOptionsDoc = attrs: + (import "${nixosPath}/lib/make-options-doc") + ({ inherit pkgs lib; } // attrs); + }; + config = config.config; + options = eval.options; + version = release; + revision = "release-${release}"; + prefix = modulesPath; + }; +in + docs.optionsNix diff --git a/nixpkgs/nixos/lib/eval-config-minimal.nix b/nixpkgs/nixos/lib/eval-config-minimal.nix new file mode 100644 index 000000000000..036389121973 --- /dev/null +++ b/nixpkgs/nixos/lib/eval-config-minimal.nix @@ -0,0 +1,50 @@ + +# DO NOT IMPORT. Use nixpkgsFlake.lib.nixos, or import (nixpkgs + "/nixos/lib") +{ lib }: # read -^ + +let + + /* + Invoke NixOS. Unlike traditional NixOS, this does not include all modules. + Any such modules have to be explicitly added via the `modules` parameter, + or imported using `imports` in a module. + + A minimal module list improves NixOS evaluation performance and allows + modules to be independently usable, supporting new use cases. + + Parameters: + + modules: A list of modules that constitute the configuration. + + specialArgs: An attribute set of module arguments. Unlike + `config._module.args`, these are available for use in + `imports`. + `config._module.args` should be preferred when possible. + + Return: + + An attribute set containing `config.system.build.toplevel` among other + attributes. See `lib.evalModules` in the Nixpkgs library. + + */ + evalModules = { + prefix ? [], + modules ? [], + specialArgs ? {}, + }: + # NOTE: Regular NixOS currently does use this function! Don't break it! + # Ideally we don't diverge, unless we learn that we should. + # In other words, only the public interface of nixos.evalModules + # is experimental. + lib.evalModules { + inherit prefix modules; + class = "nixos"; + specialArgs = { + modulesPath = builtins.toString ../modules; + } // specialArgs; + }; + +in +{ + inherit evalModules; +} diff --git a/nixpkgs/nixos/lib/eval-config.nix b/nixpkgs/nixos/lib/eval-config.nix new file mode 100644 index 000000000000..da099f86aa2c --- /dev/null +++ b/nixpkgs/nixos/lib/eval-config.nix @@ -0,0 +1,116 @@ +# From an end-user configuration file (`configuration.nix'), build a NixOS +# configuration object (`config') from which we can retrieve option +# values. + +# !!! Please think twice before adding to this argument list! +# Ideally eval-config.nix would be an extremely thin wrapper +# around lib.evalModules, so that modular systems that have nixos configs +# as subcomponents (e.g. the container feature, or nixops if network +# expressions are ever made modular at the top level) can just use +# types.submodule instead of using eval-config.nix +evalConfigArgs@ +{ # !!! system can be set modularly, would be nice to remove, + # however, removing or changing this default is too much + # of a breaking change. To set it modularly, pass `null`. + system ? builtins.currentSystem +, # !!! is this argument needed any more? The pkgs argument can + # be set modularly anyway. + pkgs ? null +, # !!! what do we gain by making this configurable? + # we can add modules that are included in specialisations, regardless + # of inheritParentConfig. + baseModules ? import ../modules/module-list.nix +, # !!! See comment about args in lib/modules.nix + extraArgs ? {} +, # !!! See comment about args in lib/modules.nix + specialArgs ? {} +, modules +, modulesLocation ? (builtins.unsafeGetAttrPos "modules" evalConfigArgs).file or null +, # !!! See comment about check in lib/modules.nix + check ? true +, prefix ? [] +, lib ? import ../../lib +, extraModules ? let e = builtins.getEnv "NIXOS_EXTRA_MODULE_PATH"; + in lib.optional (e != "") (import e) +}: + +let + inherit (lib) optional; + + evalModulesMinimal = (import ./default.nix { + inherit lib; + # Implicit use of feature is noted in implementation. + featureFlags.minimalModules = { }; + }).evalModules; + + pkgsModule = rec { + _file = ./eval-config.nix; + key = _file; + config = lib.mkMerge ( + (optional (system != null) { + # Explicit `nixpkgs.system` or `nixpkgs.localSystem` should override + # this. Since the latter defaults to the former, the former should + # default to the argument. That way this new default could propagate all + # they way through, but has the last priority behind everything else. + nixpkgs.system = lib.mkDefault system; + }) + ++ + (optional (pkgs != null) { + # This should be default priority, so it conflicts with any user-defined pkgs. + nixpkgs.pkgs = pkgs; + }) + ); + }; + + withWarnings = x: + lib.warnIf (evalConfigArgs?extraArgs) "The extraArgs argument to eval-config.nix is deprecated. Please set config._module.args instead." + lib.warnIf (evalConfigArgs?check) "The check argument to eval-config.nix is deprecated. Please set config._module.check instead." + x; + + legacyModules = + lib.optional (evalConfigArgs?extraArgs) { + config = { + _module.args = extraArgs; + }; + } + ++ lib.optional (evalConfigArgs?check) { + config = { + _module.check = lib.mkDefault check; + }; + }; + + allUserModules = + let + # Add the invoking file (or specified modulesLocation) as error message location + # for modules that don't have their own locations; presumably inline modules. + locatedModules = + if modulesLocation == null then + modules + else + map (lib.setDefaultModuleLocation modulesLocation) modules; + in + locatedModules ++ legacyModules; + + noUserModules = evalModulesMinimal ({ + inherit prefix specialArgs; + modules = baseModules ++ extraModules ++ [ pkgsModule modulesModule ]; + }); + + # Extra arguments that are useful for constructing a similar configuration. + modulesModule = { + config = { + _module.args = { + inherit noUserModules baseModules extraModules modules; + }; + }; + }; + + nixosWithUserModules = noUserModules.extendModules { modules = allUserModules; }; + + withExtraAttrs = configuration: configuration // { + inherit extraArgs; + inherit (configuration._module.args) pkgs; + extendModules = args: withExtraAttrs (configuration.extendModules args); + }; +in +withWarnings (withExtraAttrs nixosWithUserModules) diff --git a/nixpkgs/nixos/lib/from-env.nix b/nixpkgs/nixos/lib/from-env.nix new file mode 100644 index 000000000000..6bd71e40e9a1 --- /dev/null +++ b/nixpkgs/nixos/lib/from-env.nix @@ -0,0 +1,4 @@ +# TODO: remove this file. There is lib.maybeEnv now +name: default: +let value = builtins.getEnv name; in +if value == "" then default else value diff --git a/nixpkgs/nixos/lib/make-btrfs-fs.nix b/nixpkgs/nixos/lib/make-btrfs-fs.nix new file mode 100644 index 000000000000..277ff6a4dca8 --- /dev/null +++ b/nixpkgs/nixos/lib/make-btrfs-fs.nix @@ -0,0 +1,67 @@ +# Builds an btrfs image containing a populated /nix/store with the closure +# of store paths passed in the storePaths parameter, in addition to the +# contents of a directory that can be populated with commands. The +# generated image is sized to only fit its contents, with the expectation +# that a script resizes the filesystem at boot time. +{ pkgs +, lib +# List of derivations to be included +, storePaths +# Whether or not to compress the resulting image with zstd +, compressImage ? false, zstd +# Shell commands to populate the ./files directory. +# All files in that directory are copied to the root of the FS. +, populateImageCommands ? "" +, volumeLabel +, uuid ? "44444444-4444-4444-8888-888888888888" +, btrfs-progs +, libfaketime +, fakeroot +}: + +let + sdClosureInfo = pkgs.buildPackages.closureInfo { rootPaths = storePaths; }; +in +pkgs.stdenv.mkDerivation { + name = "btrfs-fs.img${lib.optionalString compressImage ".zst"}"; + + nativeBuildInputs = [ btrfs-progs libfaketime fakeroot ] ++ lib.optional compressImage zstd; + + buildCommand = + '' + ${if compressImage then "img=temp.img" else "img=$out"} + + set -x + ( + mkdir -p ./files + ${populateImageCommands} + ) + + mkdir -p ./rootImage/nix/store + + xargs -I % cp -a --reflink=auto % -t ./rootImage/nix/store/ < ${sdClosureInfo}/store-paths + ( + GLOBIGNORE=".:.." + shopt -u dotglob + + for f in ./files/*; do + cp -a --reflink=auto -t ./rootImage/ "$f" + done + ) + + cp ${sdClosureInfo}/registration ./rootImage/nix-path-registration + + touch $img + faketime -f "1970-01-01 00:00:01" fakeroot mkfs.btrfs -L ${volumeLabel} -U ${uuid} -r ./rootImage --shrink $img + + if ! btrfs check $img; then + echo "--- 'btrfs check' failed for BTRFS image ---" + return 1 + fi + + if [ ${builtins.toString compressImage} ]; then + echo "Compressing image" + zstd -v --no-progress ./$img -o $out + fi + ''; +} diff --git a/nixpkgs/nixos/lib/make-channel.nix b/nixpkgs/nixos/lib/make-channel.nix new file mode 100644 index 000000000000..0a511468fb2d --- /dev/null +++ b/nixpkgs/nixos/lib/make-channel.nix @@ -0,0 +1,31 @@ +/* Build a channel tarball. These contain, in addition to the nixpkgs + * expressions themselves, files that indicate the version of nixpkgs + * that they represent. + */ +{ pkgs, nixpkgs, version, versionSuffix }: + +pkgs.releaseTools.makeSourceTarball { + name = "nixos-channel"; + + src = nixpkgs; + + officialRelease = false; # FIXME: fix this in makeSourceTarball + inherit version versionSuffix; + + buildInputs = [ pkgs.nix ]; + + distPhase = '' + rm -rf .git + echo -n $VERSION_SUFFIX > .version-suffix + echo -n ${nixpkgs.rev or nixpkgs.shortRev} > .git-revision + releaseName=nixos-$VERSION$VERSION_SUFFIX + mkdir -p $out/tarballs + cp -prd . ../$releaseName + chmod -R u+w ../$releaseName + ln -s . ../$releaseName/nixpkgs # hack to make ‘<nixpkgs>’ work + NIX_STATE_DIR=$TMPDIR nix-env -f ../$releaseName/default.nix -qaP --meta --show-trace --xml \* > /dev/null + cd .. + chmod -R u+w $releaseName + tar cfJ $out/tarballs/$releaseName.tar.xz $releaseName + ''; +} diff --git a/nixpkgs/nixos/lib/make-disk-image.nix b/nixpkgs/nixos/lib/make-disk-image.nix new file mode 100644 index 000000000000..1a33abd01ea1 --- /dev/null +++ b/nixpkgs/nixos/lib/make-disk-image.nix @@ -0,0 +1,625 @@ +/* Technical details + +`make-disk-image` has a bit of magic to minimize the amount of work to do in a virtual machine. + +It relies on the [LKL (Linux Kernel Library) project](https://github.com/lkl/linux) which provides Linux kernel as userspace library. + +The Nix-store only image only need to run LKL tools to produce an image and will never spawn a virtual machine, whereas full images will always require a virtual machine, but also use LKL. + +### Image preparation phase + +Image preparation phase will produce the initial image layout in a folder: + +- devise a root folder based on `$PWD` +- prepare the contents by copying and restoring ACLs in this root folder +- load in the Nix store database all additional paths computed by `pkgs.closureInfo` in a temporary Nix store +- run `nixos-install` in a temporary folder +- transfer from the temporary store the additional paths registered to the installed NixOS +- compute the size of the disk image based on the apparent size of the root folder +- partition the disk image using the corresponding script according to the partition table type +- format the partitions if needed +- use `cptofs` (LKL tool) to copy the root folder inside the disk image + +At this step, the disk image already contains the Nix store, it now only needs to be converted to the desired format to be used. + +### Image conversion phase + +Using `qemu-img`, the disk image is converted from a raw format to the desired format: qcow2(-compressed), vdi, vpc. + +### Image Partitioning + +#### `none` + +No partition table layout is written. The image is a bare filesystem image. + +#### `legacy` + +The image is partitioned using MBR. There is one primary ext4 partition starting at 1 MiB that fills the rest of the disk image. + +This partition layout is unsuitable for UEFI. + +#### `legacy+gpt` + +This partition table type uses GPT and: + +- create a "no filesystem" partition from 1MiB to 2MiB ; +- set `bios_grub` flag on this "no filesystem" partition, which marks it as a [GRUB BIOS partition](https://www.gnu.org/software/parted/manual/html_node/set.html) ; +- create a primary ext4 partition starting at 2MiB and extending to the full disk image ; +- perform optimal alignments checks on each partition + +This partition layout is unsuitable for UEFI boot, because it has no ESP (EFI System Partition) partition. It can work with CSM (Compatibility Support Module) which emulates legacy (BIOS) boot for UEFI. + +#### `efi` + +This partition table type uses GPT and: + +- creates an FAT32 ESP partition from 8MiB to specified `bootSize` parameter (256MiB by default), set it bootable ; +- creates an primary ext4 partition starting after the boot partition and extending to the full disk image + +#### `hybrid` + +This partition table type uses GPT and: + +- creates a "no filesystem" partition from 0 to 1MiB, set `bios_grub` flag on it ; +- creates an FAT32 ESP partition from 8MiB to specified `bootSize` parameter (256MiB by default), set it bootable ; +- creates a primary ext4 partition starting after the boot one and extending to the full disk image + +This partition could be booted by a BIOS able to understand GPT layouts and recognizing the MBR at the start. + +### How to run determinism analysis on results? + +Build your derivation with `--check` to rebuild it and verify it is the same. + +If it fails, you will be left with two folders with one having `.check`. + +You can use `diffoscope` to see the differences between the folders. + +However, `diffoscope` is currently not able to diff two QCOW2 filesystems, thus, it is advised to use raw format. + +Even if you use raw disks, `diffoscope` cannot diff the partition table and partitions recursively. + +To solve this, you can run `fdisk -l $image` and generate `dd if=$image of=$image-p$i.raw skip=$start count=$sectors` for each `(start, sectors)` listed in the `fdisk` output. Now, you will have each partition as a separate file and you can compare them in pairs. +*/ +{ pkgs +, lib + +, # The NixOS configuration to be installed onto the disk image. + config + +, # The size of the disk, in megabytes. + # if "auto" size is calculated based on the contents copied to it and + # additionalSpace is taken into account. + diskSize ? "auto" + +, # additional disk space to be added to the image if diskSize "auto" + # is used + additionalSpace ? "512M" + +, # size of the boot partition, is only used if partitionTableType is + # either "efi" or "hybrid" + # This will be undersized slightly, as this is actually the offset of + # the end of the partition. Generally it will be 1MiB smaller. + bootSize ? "256M" + +, # The files and directories to be placed in the target file system. + # This is a list of attribute sets {source, target, mode, user, group} where + # `source' is the file system object (regular file or directory) to be + # grafted in the file system at path `target', `mode' is a string containing + # the permissions that will be set (ex. "755"), `user' and `group' are the + # user and group name that will be set as owner of the files. + # `mode', `user', and `group' are optional. + # When setting one of `user' or `group', the other needs to be set too. + contents ? [] + +, # Type of partition table to use; either "legacy", "efi", or "none". + # For "efi" images, the GPT partition table is used and a mandatory ESP + # partition of reasonable size is created in addition to the root partition. + # For "legacy", the msdos partition table is used and a single large root + # partition is created. + # For "legacy+gpt", the GPT partition table is used, a 1MiB no-fs partition for + # use by the bootloader is created, and a single large root partition is + # created. + # For "hybrid", the GPT partition table is used and a mandatory ESP + # partition of reasonable size is created in addition to the root partition. + # Also a legacy MBR will be present. + # For "none", no partition table is created. Enabling `installBootLoader` + # most likely fails as GRUB will probably refuse to install. + partitionTableType ? "legacy" + +, # Whether to invoke `switch-to-configuration boot` during image creation + installBootLoader ? true + +, # Whether to output have EFIVARS available in $out/efi-vars.fd and use it during disk creation + touchEFIVars ? false + +, # OVMF firmware derivation + OVMF ? pkgs.OVMF.fd + +, # EFI firmware + efiFirmware ? OVMF.firmware + +, # EFI variables + efiVariables ? OVMF.variables + +, # The root file system type. + fsType ? "ext4" + +, # Filesystem label + label ? if onlyNixStore then "nix-store" else "nixos" + +, # The initial NixOS configuration file to be copied to + # /etc/nixos/configuration.nix. + configFile ? null + +, # Shell code executed after the VM has finished. + postVM ? "" + +, # Guest memory size + memSize ? 1024 + +, # Copy the contents of the Nix store to the root of the image and + # skip further setup. Incompatible with `contents`, + # `installBootLoader` and `configFile`. + onlyNixStore ? false + +, name ? "nixos-disk-image" + +, # Disk image format, one of qcow2, qcow2-compressed, vdi, vpc, raw. + format ? "raw" + + # Whether to fix: + # - GPT Disk Unique Identifier (diskGUID) + # - GPT Partition Unique Identifier: depends on the layout, root partition UUID can be controlled through `rootGPUID` option + # - GPT Partition Type Identifier: fixed according to the layout, e.g. ESP partition, etc. through `parted` invocation. + # - Filesystem Unique Identifier when fsType = ext4 for *root partition*. + # BIOS/MBR support is "best effort" at the moment. + # Boot partitions may not be deterministic. + # Also, to fix last time checked of the ext4 partition if fsType = ext4. +, deterministic ? true + + # GPT Partition Unique Identifier for root partition. +, rootGPUID ? "F222513B-DED1-49FA-B591-20CE86A2FE7F" + # When fsType = ext4, this is the root Filesystem Unique Identifier. + # TODO: support other filesystems someday. +, rootFSUID ? (if fsType == "ext4" then rootGPUID else null) + +, # Whether a nix channel based on the current source tree should be + # made available inside the image. Useful for interactive use of nix + # utils, but changes the hash of the image when the sources are + # updated. + copyChannel ? true + +, # Additional store paths to copy to the image's store. + additionalPaths ? [] +}: + +assert (lib.assertOneOf "partitionTableType" partitionTableType [ "legacy" "legacy+gpt" "efi" "hybrid" "none" ]); +assert (lib.assertMsg (fsType == "ext4" && deterministic -> rootFSUID != null) "In deterministic mode with a ext4 partition, rootFSUID must be non-null, by default, it is equal to rootGPUID."); + # We use -E offset=X below, which is only supported by e2fsprogs +assert (lib.assertMsg (partitionTableType != "none" -> fsType == "ext4") "to produce a partition table, we need to use -E offset flag which is support only for fsType = ext4"); +assert (lib.assertMsg (touchEFIVars -> partitionTableType == "hybrid" || partitionTableType == "efi" || partitionTableType == "legacy+gpt") "EFI variables can be used only with a partition table of type: hybrid, efi or legacy+gpt."); + # If only Nix store image, then: contents must be empty, configFile must be unset, and we should no install bootloader. +assert (lib.assertMsg (onlyNixStore -> contents == [] && configFile == null && !installBootLoader) "In a only Nix store image, the contents must be empty, no configuration must be provided and no bootloader should be installed."); +# Either both or none of {user,group} need to be set +assert (lib.assertMsg (lib.all + (attrs: ((attrs.user or null) == null) + == ((attrs.group or null) == null)) + contents) "Contents of the disk image should set none of {user, group} or both at the same time."); + +with lib; + +let format' = format; in let + + format = if format' == "qcow2-compressed" then "qcow2" else format'; + + compress = optionalString (format' == "qcow2-compressed") "-c"; + + filename = "nixos." + { + qcow2 = "qcow2"; + vdi = "vdi"; + vpc = "vhd"; + raw = "img"; + }.${format} or format; + + rootPartition = { # switch-case + legacy = "1"; + "legacy+gpt" = "2"; + efi = "2"; + hybrid = "3"; + }.${partitionTableType}; + + partitionDiskScript = { # switch-case + legacy = '' + parted --script $diskImage -- \ + mklabel msdos \ + mkpart primary ext4 1MiB -1 + ''; + "legacy+gpt" = '' + parted --script $diskImage -- \ + mklabel gpt \ + mkpart no-fs 1MB 2MB \ + set 1 bios_grub on \ + align-check optimal 1 \ + mkpart primary ext4 2MB -1 \ + align-check optimal 2 \ + print + ${optionalString deterministic '' + sgdisk \ + --disk-guid=97FD5997-D90B-4AA3-8D16-C1723AEA73C \ + --partition-guid=1:1C06F03B-704E-4657-B9CD-681A087A2FDC \ + --partition-guid=2:970C694F-AFD0-4B99-B750-CDB7A329AB6F \ + --partition-guid=3:${rootGPUID} \ + $diskImage + ''} + ''; + efi = '' + parted --script $diskImage -- \ + mklabel gpt \ + mkpart ESP fat32 8MiB ${bootSize} \ + set 1 boot on \ + mkpart primary ext4 ${bootSize} -1 + ${optionalString deterministic '' + sgdisk \ + --disk-guid=97FD5997-D90B-4AA3-8D16-C1723AEA73C \ + --partition-guid=1:1C06F03B-704E-4657-B9CD-681A087A2FDC \ + --partition-guid=2:${rootGPUID} \ + $diskImage + ''} + ''; + hybrid = '' + parted --script $diskImage -- \ + mklabel gpt \ + mkpart ESP fat32 8MiB ${bootSize} \ + set 1 boot on \ + mkpart no-fs 0 1024KiB \ + set 2 bios_grub on \ + mkpart primary ext4 ${bootSize} -1 + ${optionalString deterministic '' + sgdisk \ + --disk-guid=97FD5997-D90B-4AA3-8D16-C1723AEA73C \ + --partition-guid=1:1C06F03B-704E-4657-B9CD-681A087A2FDC \ + --partition-guid=2:970C694F-AFD0-4B99-B750-CDB7A329AB6F \ + --partition-guid=3:${rootGPUID} \ + $diskImage + ''} + ''; + none = ""; + }.${partitionTableType}; + + useEFIBoot = touchEFIVars; + + nixpkgs = cleanSource pkgs.path; + + # FIXME: merge with channel.nix / make-channel.nix. + channelSources = pkgs.runCommand "nixos-${config.system.nixos.version}" {} '' + mkdir -p $out + cp -prd ${nixpkgs.outPath} $out/nixos + chmod -R u+w $out/nixos + if [ ! -e $out/nixos/nixpkgs ]; then + ln -s . $out/nixos/nixpkgs + fi + rm -rf $out/nixos/.git + echo -n ${config.system.nixos.versionSuffix} > $out/nixos/.version-suffix + ''; + + binPath = with pkgs; makeBinPath ( + [ rsync + util-linux + parted + e2fsprogs + lkl + config.system.build.nixos-install + config.system.build.nixos-enter + nix + systemdMinimal + ] + ++ lib.optional deterministic gptfdisk + ++ stdenv.initialPath); + + # I'm preserving the line below because I'm going to search for it across nixpkgs to consolidate + # image building logic. The comment right below this now appears in 4 different places in nixpkgs :) + # !!! should use XML. + sources = map (x: x.source) contents; + targets = map (x: x.target) contents; + modes = map (x: x.mode or "''") contents; + users = map (x: x.user or "''") contents; + groups = map (x: x.group or "''") contents; + + basePaths = [ config.system.build.toplevel ] + ++ lib.optional copyChannel channelSources; + + additionalPaths' = subtractLists basePaths additionalPaths; + + closureInfo = pkgs.closureInfo { + rootPaths = basePaths ++ additionalPaths'; + }; + + blockSize = toString (4 * 1024); # ext4fs block size (not block device sector size) + + prepareImage = '' + export PATH=${binPath} + + # Yes, mkfs.ext4 takes different units in different contexts. Fun. + sectorsToKilobytes() { + echo $(( ( "$1" * 512 ) / 1024 )) + } + + sectorsToBytes() { + echo $(( "$1" * 512 )) + } + + # Given lines of numbers, adds them together + sum_lines() { + local acc=0 + while read -r number; do + acc=$((acc+number)) + done + echo "$acc" + } + + mebibyte=$(( 1024 * 1024 )) + + # Approximative percentage of reserved space in an ext4 fs over 512MiB. + # 0.05208587646484375 + # × 1000, integer part: 52 + compute_fudge() { + echo $(( $1 * 52 / 1000 )) + } + + mkdir $out + + root="$PWD/root" + mkdir -p $root + + # Copy arbitrary other files into the image + # Semi-shamelessly copied from make-etc.sh. I (@copumpkin) shall factor this stuff out as part of + # https://github.com/NixOS/nixpkgs/issues/23052. + set -f + sources_=(${concatStringsSep " " sources}) + targets_=(${concatStringsSep " " targets}) + modes_=(${concatStringsSep " " modes}) + set +f + + for ((i = 0; i < ''${#targets_[@]}; i++)); do + source="''${sources_[$i]}" + target="''${targets_[$i]}" + mode="''${modes_[$i]}" + + if [ -n "$mode" ]; then + rsync_chmod_flags="--chmod=$mode" + else + rsync_chmod_flags="" + fi + # Unfortunately cptofs only supports modes, not ownership, so we can't use + # rsync's --chown option. Instead, we change the ownerships in the + # VM script with chown. + rsync_flags="-a --no-o --no-g $rsync_chmod_flags" + if [[ "$source" =~ '*' ]]; then + # If the source name contains '*', perform globbing. + mkdir -p $root/$target + for fn in $source; do + rsync $rsync_flags "$fn" $root/$target/ + done + else + mkdir -p $root/$(dirname $target) + if [ -e $root/$target ]; then + echo "duplicate entry $target -> $source" + exit 1 + elif [ -d $source ]; then + # Append a slash to the end of source to get rsync to copy the + # directory _to_ the target instead of _inside_ the target. + # (See `man rsync`'s note on a trailing slash.) + rsync $rsync_flags $source/ $root/$target + else + rsync $rsync_flags $source $root/$target + fi + fi + done + + export HOME=$TMPDIR + + # Provide a Nix database so that nixos-install can copy closures. + export NIX_STATE_DIR=$TMPDIR/state + nix-store --load-db < ${closureInfo}/registration + + chmod 755 "$TMPDIR" + echo "running nixos-install..." + nixos-install --root $root --no-bootloader --no-root-passwd \ + --system ${config.system.build.toplevel} \ + ${if copyChannel then "--channel ${channelSources}" else "--no-channel-copy"} \ + --substituters "" + + ${optionalString (additionalPaths' != []) '' + nix --extra-experimental-features nix-command copy --to $root --no-check-sigs ${concatStringsSep " " additionalPaths'} + ''} + + diskImage=nixos.raw + + ${if diskSize == "auto" then '' + ${if partitionTableType == "efi" || partitionTableType == "hybrid" then '' + # Add the GPT at the end + gptSpace=$(( 512 * 34 * 1 )) + # Normally we'd need to account for alignment and things, if bootSize + # represented the actual size of the boot partition. But it instead + # represents the offset at which it ends. + # So we know bootSize is the reserved space in front of the partition. + reservedSpace=$(( gptSpace + $(numfmt --from=iec '${bootSize}') )) + '' else if partitionTableType == "legacy+gpt" then '' + # Add the GPT at the end + gptSpace=$(( 512 * 34 * 1 )) + # And include the bios_grub partition; the ext4 partition starts at 2MB exactly. + reservedSpace=$(( gptSpace + 2 * mebibyte )) + '' else if partitionTableType == "legacy" then '' + # Add the 1MiB aligned reserved space (includes MBR) + reservedSpace=$(( mebibyte )) + '' else '' + reservedSpace=0 + ''} + additionalSpace=$(( $(numfmt --from=iec '${additionalSpace}') + reservedSpace )) + + # Compute required space in filesystem blocks + diskUsage=$(find . ! -type d -print0 | du --files0-from=- --apparent-size --block-size "${blockSize}" | cut -f1 | sum_lines) + # Each inode takes space! + numInodes=$(find . | wc -l) + # Convert to bytes, inodes take two blocks each! + diskUsage=$(( (diskUsage + 2 * numInodes) * ${blockSize} )) + # Then increase the required space to account for the reserved blocks. + fudge=$(compute_fudge $diskUsage) + requiredFilesystemSpace=$(( diskUsage + fudge )) + + diskSize=$(( requiredFilesystemSpace + additionalSpace )) + + # Round up to the nearest mebibyte. + # This ensures whole 512 bytes sector sizes in the disk image + # and helps towards aligning partitions optimally. + if (( diskSize % mebibyte )); then + diskSize=$(( ( diskSize / mebibyte + 1) * mebibyte )) + fi + + truncate -s "$diskSize" $diskImage + + printf "Automatic disk size...\n" + printf " Closure space use: %d bytes\n" $diskUsage + printf " fudge: %d bytes\n" $fudge + printf " Filesystem size needed: %d bytes\n" $requiredFilesystemSpace + printf " Additional space: %d bytes\n" $additionalSpace + printf " Disk image size: %d bytes\n" $diskSize + '' else '' + truncate -s ${toString diskSize}M $diskImage + ''} + + ${partitionDiskScript} + + ${if partitionTableType != "none" then '' + # Get start & length of the root partition in sectors to $START and $SECTORS. + eval $(partx $diskImage -o START,SECTORS --nr ${rootPartition} --pairs) + + mkfs.${fsType} -b ${blockSize} -F -L ${label} $diskImage -E offset=$(sectorsToBytes $START) $(sectorsToKilobytes $SECTORS)K + '' else '' + mkfs.${fsType} -b ${blockSize} -F -L ${label} $diskImage + ''} + + echo "copying staging root to image..." + cptofs -p ${optionalString (partitionTableType != "none") "-P ${rootPartition}"} \ + -t ${fsType} \ + -i $diskImage \ + $root${optionalString onlyNixStore builtins.storeDir}/* / || + (echo >&2 "ERROR: cptofs failed. diskSize might be too small for closure."; exit 1) + ''; + + moveOrConvertImage = '' + ${if format == "raw" then '' + mv $diskImage $out/${filename} + '' else '' + ${pkgs.qemu-utils}/bin/qemu-img convert -f raw -O ${format} ${compress} $diskImage $out/${filename} + ''} + diskImage=$out/${filename} + ''; + + createEFIVars = '' + efiVars=$out/efi-vars.fd + cp ${efiVariables} $efiVars + chmod 0644 $efiVars + ''; + + createHydraBuildProducts = '' + mkdir -p $out/nix-support + echo "file ${format}-image $out/${filename}" >> $out/nix-support/hydra-build-products + ''; + + buildImage = pkgs.vmTools.runInLinuxVM ( + pkgs.runCommand name { + preVM = prepareImage + lib.optionalString touchEFIVars createEFIVars; + buildInputs = with pkgs; [ util-linux e2fsprogs dosfstools ]; + postVM = moveOrConvertImage + createHydraBuildProducts + postVM; + QEMU_OPTS = + concatStringsSep " " (lib.optional useEFIBoot "-drive if=pflash,format=raw,unit=0,readonly=on,file=${efiFirmware}" + ++ lib.optionals touchEFIVars [ + "-drive if=pflash,format=raw,unit=1,file=$efiVars" + ] + ); + inherit memSize; + } '' + export PATH=${binPath}:$PATH + + rootDisk=${if partitionTableType != "none" then "/dev/vda${rootPartition}" else "/dev/vda"} + + # It is necessary to set root filesystem unique identifier in advance, otherwise + # bootloader might get the wrong one and fail to boot. + # At the end, we reset again because we want deterministic timestamps. + ${optionalString (fsType == "ext4" && deterministic) '' + tune2fs -T now ${optionalString deterministic "-U ${rootFSUID}"} -c 0 -i 0 $rootDisk + ''} + # make systemd-boot find ESP without udev + mkdir /dev/block + ln -s /dev/vda1 /dev/block/254:1 + + mountPoint=/mnt + mkdir $mountPoint + mount $rootDisk $mountPoint + + # Create the ESP and mount it. Unlike e2fsprogs, mkfs.vfat doesn't support an + # '-E offset=X' option, so we can't do this outside the VM. + ${optionalString (partitionTableType == "efi" || partitionTableType == "hybrid") '' + mkdir -p /mnt/boot + mkfs.vfat -n ESP /dev/vda1 + mount /dev/vda1 /mnt/boot + + ${optionalString touchEFIVars "mount -t efivarfs efivarfs /sys/firmware/efi/efivars"} + ''} + + # Install a configuration.nix + mkdir -p /mnt/etc/nixos + ${optionalString (configFile != null) '' + cp ${configFile} /mnt/etc/nixos/configuration.nix + ''} + + ${lib.optionalString installBootLoader '' + # In this throwaway resource, we only have /dev/vda, but the actual VM may refer to another disk for bootloader, e.g. /dev/vdb + # Use this option to create a symlink from vda to any arbitrary device you want. + ${optionalString (config.boot.loader.grub.enable && config.boot.loader.grub.device != "/dev/vda") '' + mkdir -p $(dirname ${config.boot.loader.grub.device}) + ln -s /dev/vda ${config.boot.loader.grub.device} + ''} + + # Set up core system link, bootloader (sd-boot, GRUB, uboot, etc.), etc. + NIXOS_INSTALL_BOOTLOADER=1 nixos-enter --root $mountPoint -- /nix/var/nix/profiles/system/bin/switch-to-configuration boot + + # The above scripts will generate a random machine-id and we don't want to bake a single ID into all our images + rm -f $mountPoint/etc/machine-id + ''} + + # Set the ownerships of the contents. The modes are set in preVM. + # No globbing on targets, so no need to set -f + targets_=(${concatStringsSep " " targets}) + users_=(${concatStringsSep " " users}) + groups_=(${concatStringsSep " " groups}) + for ((i = 0; i < ''${#targets_[@]}; i++)); do + target="''${targets_[$i]}" + user="''${users_[$i]}" + group="''${groups_[$i]}" + if [ -n "$user$group" ]; then + # We have to nixos-enter since we need to use the user and group of the VM + nixos-enter --root $mountPoint -- chown -R "$user:$group" "$target" + fi + done + + umount -R /mnt + + # Make sure resize2fs works. Note that resize2fs has stricter criteria for resizing than a normal + # mount, so the `-c 0` and `-i 0` don't affect it. Setting it to `now` doesn't produce deterministic + # output, of course, but we can fix that when/if we start making images deterministic. + # In deterministic mode, this is fixed to 1970-01-01 (UNIX timestamp 0). + # This two-step approach is necessary otherwise `tune2fs` will want a fresher filesystem to perform + # some changes. + ${optionalString (fsType == "ext4") '' + tune2fs -T now ${optionalString deterministic "-U ${rootFSUID}"} -c 0 -i 0 $rootDisk + ${optionalString deterministic "tune2fs -f -T 19700101 $rootDisk"} + ''} + '' + ); +in + if onlyNixStore then + pkgs.runCommand name {} + (prepareImage + moveOrConvertImage + createHydraBuildProducts + postVM) + else buildImage diff --git a/nixpkgs/nixos/lib/make-ext4-fs.nix b/nixpkgs/nixos/lib/make-ext4-fs.nix new file mode 100644 index 000000000000..b8e1b8d24c48 --- /dev/null +++ b/nixpkgs/nixos/lib/make-ext4-fs.nix @@ -0,0 +1,95 @@ +# Builds an ext4 image containing a populated /nix/store with the closure +# of store paths passed in the storePaths parameter, in addition to the +# contents of a directory that can be populated with commands. The +# generated image is sized to only fit its contents, with the expectation +# that a script resizes the filesystem at boot time. +{ pkgs +, lib +# List of derivations to be included +, storePaths +# Whether or not to compress the resulting image with zstd +, compressImage ? false, zstd +# Shell commands to populate the ./files directory. +# All files in that directory are copied to the root of the FS. +, populateImageCommands ? "" +, volumeLabel +, uuid ? "44444444-4444-4444-8888-888888888888" +, e2fsprogs +, libfaketime +, perl +, fakeroot +}: + +let + sdClosureInfo = pkgs.buildPackages.closureInfo { rootPaths = storePaths; }; +in +pkgs.stdenv.mkDerivation { + name = "ext4-fs.img${lib.optionalString compressImage ".zst"}"; + + nativeBuildInputs = [ e2fsprogs.bin libfaketime perl fakeroot ] + ++ lib.optional compressImage zstd; + + buildCommand = + '' + ${if compressImage then "img=temp.img" else "img=$out"} + ( + mkdir -p ./files + ${populateImageCommands} + ) + + echo "Preparing store paths for image..." + + # Create nix/store before copying path + mkdir -p ./rootImage/nix/store + + xargs -I % cp -a --reflink=auto % -t ./rootImage/nix/store/ < ${sdClosureInfo}/store-paths + ( + GLOBIGNORE=".:.." + shopt -u dotglob + + for f in ./files/*; do + cp -a --reflink=auto -t ./rootImage/ "$f" + done + ) + + # Also include a manifest of the closures in a format suitable for nix-store --load-db + cp ${sdClosureInfo}/registration ./rootImage/nix-path-registration + + # Make a crude approximation of the size of the target image. + # If the script starts failing, increase the fudge factors here. + numInodes=$(find ./rootImage | wc -l) + numDataBlocks=$(du -s -c -B 4096 --apparent-size ./rootImage | tail -1 | awk '{ print int($1 * 1.10) }') + bytes=$((2 * 4096 * $numInodes + 4096 * $numDataBlocks)) + echo "Creating an EXT4 image of $bytes bytes (numInodes=$numInodes, numDataBlocks=$numDataBlocks)" + + truncate -s $bytes $img + + faketime -f "1970-01-01 00:00:01" fakeroot mkfs.ext4 -L ${volumeLabel} -U ${uuid} -d ./rootImage $img + + export EXT2FS_NO_MTAB_OK=yes + # I have ended up with corrupted images sometimes, I suspect that happens when the build machine's disk gets full during the build. + if ! fsck.ext4 -n -f $img; then + echo "--- Fsck failed for EXT4 image of $bytes bytes (numInodes=$numInodes, numDataBlocks=$numDataBlocks) ---" + cat errorlog + return 1 + fi + + # We may want to shrink the file system and resize the image to + # get rid of the unnecessary slack here--but see + # https://github.com/NixOS/nixpkgs/issues/125121 for caveats. + + # shrink to fit + resize2fs -M $img + + # Add 16 MebiByte to the current_size + new_size=$(dumpe2fs -h $img | awk -F: \ + '/Block count/{count=$2} /Block size/{size=$2} END{print (count*size+16*2**20)/size}') + + resize2fs $img $new_size + + if [ ${builtins.toString compressImage} ]; then + echo "Compressing image" + zstd -v --no-progress ./$img -o $out + fi + ''; +} diff --git a/nixpkgs/nixos/lib/make-iso9660-image.nix b/nixpkgs/nixos/lib/make-iso9660-image.nix new file mode 100644 index 000000000000..2f7dcf519a16 --- /dev/null +++ b/nixpkgs/nixos/lib/make-iso9660-image.nix @@ -0,0 +1,65 @@ +{ stdenv, closureInfo, xorriso, syslinux, libossp_uuid + +, # The file name of the resulting ISO image. + isoName ? "cd.iso" + +, # The files and directories to be placed in the ISO file system. + # This is a list of attribute sets {source, target} where `source' + # is the file system object (regular file or directory) to be + # grafted in the file system at path `target'. + contents + +, # In addition to `contents', the closure of the store paths listed + # in `storeContents' are also placed in the Nix store of the CD. + # This is a list of attribute sets {object, symlink} where `object' + # is a store path whose closure will be copied, and `symlink' is a + # symlink to `object' that will be added to the CD. + storeContents ? [] + +, # Whether this should be an El-Torito bootable CD. + bootable ? false + +, # Whether this should be an efi-bootable El-Torito CD. + efiBootable ? false + +, # Whether this should be an hybrid CD (bootable from USB as well as CD). + usbBootable ? false + +, # The path (in the ISO file system) of the boot image. + bootImage ? "" + +, # The path (in the ISO file system) of the efi boot image. + efiBootImage ? "" + +, # The path (outside the ISO file system) of the isohybrid-mbr image. + isohybridMbrImage ? "" + +, # Whether to compress the resulting ISO image with zstd. + compressImage ? false, zstd + +, # The volume ID. + volumeID ? "" +}: + +assert bootable -> bootImage != ""; +assert efiBootable -> efiBootImage != ""; +assert usbBootable -> isohybridMbrImage != ""; + +stdenv.mkDerivation { + name = isoName; + __structuredAttrs = true; + + buildCommandPath = ./make-iso9660-image.sh; + nativeBuildInputs = [ xorriso syslinux zstd libossp_uuid ]; + + inherit isoName bootable bootImage compressImage volumeID efiBootImage efiBootable isohybridMbrImage usbBootable; + + sources = map (x: x.source) contents; + targets = map (x: x.target) contents; + + objects = map (x: x.object) storeContents; + symlinks = map (x: x.symlink) storeContents; + + # For obtaining the closure of `storeContents'. + closureInfo = closureInfo { rootPaths = map (x: x.object) storeContents; }; +} diff --git a/nixpkgs/nixos/lib/make-iso9660-image.sh b/nixpkgs/nixos/lib/make-iso9660-image.sh new file mode 100644 index 000000000000..34febe9cfe0e --- /dev/null +++ b/nixpkgs/nixos/lib/make-iso9660-image.sh @@ -0,0 +1,130 @@ +# Remove the initial slash from a path, since genisofs likes it that way. +stripSlash() { + res="$1" + if test "${res:0:1}" = /; then res=${res:1}; fi +} + +# Escape potential equal signs (=) with backslash (\=) +escapeEquals() { + echo "$1" | sed -e 's/\\/\\\\/g' -e 's/=/\\=/g' +} + +# Queues an file/directory to be placed on the ISO. +# An entry consists of a local source path (2) and +# a destination path on the ISO (1). +addPath() { + target="$1" + source="$2" + echo "$(escapeEquals "$target")=$(escapeEquals "$source")" >> pathlist +} + +stripSlash "$bootImage"; bootImage="$res" + + +if test -n "$bootable"; then + + # The -boot-info-table option modifies the $bootImage file, so + # find it in `contents' and make a copy of it (since the original + # is read-only in the Nix store...). + for ((i = 0; i < ${#targets[@]}; i++)); do + stripSlash "${targets[$i]}" + if test "$res" = "$bootImage"; then + echo "copying the boot image ${sources[$i]}" + cp "${sources[$i]}" boot.img + chmod u+w boot.img + sources[$i]=boot.img + fi + done + + isoBootFlags="-eltorito-boot ${bootImage} + -eltorito-catalog .boot.cat + -no-emul-boot -boot-load-size 4 -boot-info-table + --sort-weight 1 /isolinux" # Make sure isolinux is near the beginning of the ISO +fi + +if test -n "$usbBootable"; then + usbBootFlags="-isohybrid-mbr ${isohybridMbrImage}" +fi + +if test -n "$efiBootable"; then + efiBootFlags="-eltorito-alt-boot + -e $efiBootImage + -no-emul-boot + -isohybrid-gpt-basdat" +fi + +touch pathlist + + +# Add the individual files. +for ((i = 0; i < ${#targets[@]}; i++)); do + stripSlash "${targets[$i]}" + addPath "$res" "${sources[$i]}" +done + + +# Add the closures of the top-level store objects. +for i in $(< $closureInfo/store-paths); do + addPath "${i:1}" "$i" +done + + +# Also include a manifest of the closures in a format suitable for +# nix-store --load-db. +if [[ ${#objects[*]} != 0 ]]; then + cp $closureInfo/registration nix-path-registration + addPath "nix-path-registration" "nix-path-registration" +fi + + +# Add symlinks to the top-level store objects. +for ((n = 0; n < ${#objects[*]}; n++)); do + object=${objects[$n]} + symlink=${symlinks[$n]} + if test "$symlink" != "none"; then + mkdir -p $(dirname ./$symlink) + ln -s $object ./$symlink + addPath "$symlink" "./$symlink" + fi +done + +mkdir -p $out/iso + +# daed2280-b91e-42c0-aed6-82c825ca41f3 is an arbitrary namespace, to prevent +# independent applications from generating the same UUID for the same value. +# (the chance of that being problematic seem pretty slim here, but that's how +# version-5 UUID's work) +xorriso="xorriso + -boot_image any gpt_disk_guid=$(uuid -v 5 daed2280-b91e-42c0-aed6-82c825ca41f3 $out | tr -d -) + -volume_date all_file_dates =$SOURCE_DATE_EPOCH + -as mkisofs + -iso-level 3 + -volid ${volumeID} + -appid nixos + -publisher nixos + -graft-points + -full-iso9660-filenames + -joliet + ${isoBootFlags} + ${usbBootFlags} + ${efiBootFlags} + -r + -path-list pathlist + --sort-weight 0 / +" + +$xorriso -output $out/iso/$isoName + +if test -n "$compressImage"; then + echo "Compressing image..." + zstd -T$NIX_BUILD_CORES --rm $out/iso/$isoName +fi + +mkdir -p $out/nix-support +echo $system > $out/nix-support/system + +if test -n "$compressImage"; then + echo "file iso $out/iso/$isoName.zst" >> $out/nix-support/hydra-build-products +else + echo "file iso $out/iso/$isoName" >> $out/nix-support/hydra-build-products +fi diff --git a/nixpkgs/nixos/lib/make-multi-disk-zfs-image.nix b/nixpkgs/nixos/lib/make-multi-disk-zfs-image.nix new file mode 100644 index 000000000000..077bb8f22707 --- /dev/null +++ b/nixpkgs/nixos/lib/make-multi-disk-zfs-image.nix @@ -0,0 +1,330 @@ +# Note: This is a private API, internal to NixOS. Its interface is subject +# to change without notice. +# +# The result of this builder is two disk images: +# +# * `boot` - a small disk formatted with FAT to be used for /boot. FAT is +# chosen to support EFI. +# * `root` - a larger disk with a zpool taking the entire disk. +# +# This two-disk approach is taken to satisfy ZFS's requirements for +# autoexpand. +# +# # Why doesn't autoexpand work with ZFS in a partition? +# +# When ZFS owns the whole disk doesn’t really use a partition: it has +# a marker partition at the start and a marker partition at the end of +# the disk. +# +# If ZFS is constrained to a partition, ZFS leaves expanding the partition +# up to the user. Obviously, the user may not choose to do so. +# +# Once the user expands the partition, calling zpool online -e expands the +# vdev to use the whole partition. It doesn’t happen automatically +# presumably because zed doesn’t get an event saying it’s partition grew, +# whereas it can and does get an event saying the whole disk it is on is +# now larger. +{ lib +, pkgs +, # The NixOS configuration to be installed onto the disk image. + config + +, # size of the FAT boot disk, in megabytes. + bootSize ? 1024 + +, # The size of the root disk, in megabytes. + rootSize ? 2048 + +, # The name of the ZFS pool + rootPoolName ? "tank" + +, # zpool properties + rootPoolProperties ? { + autoexpand = "on"; + } +, # pool-wide filesystem properties + rootPoolFilesystemProperties ? { + acltype = "posixacl"; + atime = "off"; + compression = "on"; + mountpoint = "legacy"; + xattr = "sa"; + } + +, # datasets, with per-attribute options: + # mount: (optional) mount point in the VM + # properties: (optional) ZFS properties on the dataset, like filesystemProperties + # Notes: + # 1. datasets will be created from shorter to longer names as a simple topo-sort + # 2. you should define a root's dataset's mount for `/` + datasets ? { } + +, # The files and directories to be placed in the target file system. + # This is a list of attribute sets {source, target} where `source' + # is the file system object (regular file or directory) to be + # grafted in the file system at path `target'. + contents ? [] + +, # The initial NixOS configuration file to be copied to + # /etc/nixos/configuration.nix. This configuration will be embedded + # inside a configuration which includes the described ZFS fileSystems. + configFile ? null + +, # Shell code executed after the VM has finished. + postVM ? "" + +, # Guest memory size + memSize ? 1024 + +, name ? "nixos-disk-image" + +, # Disk image format, one of qcow2, qcow2-compressed, vdi, vpc, raw. + format ? "raw" + +, # Include a copy of Nixpkgs in the disk image + includeChannel ? true +}: +let + formatOpt = if format == "qcow2-compressed" then "qcow2" else format; + + compress = lib.optionalString (format == "qcow2-compressed") "-c"; + + filenameSuffix = "." + { + qcow2 = "qcow2"; + vdi = "vdi"; + vpc = "vhd"; + raw = "img"; + }.${formatOpt} or formatOpt; + bootFilename = "nixos.boot${filenameSuffix}"; + rootFilename = "nixos.root${filenameSuffix}"; + + # FIXME: merge with channel.nix / make-channel.nix. + channelSources = + let + nixpkgs = lib.cleanSource pkgs.path; + in + pkgs.runCommand "nixos-${config.system.nixos.version}" {} '' + mkdir -p $out + cp -prd ${nixpkgs.outPath} $out/nixos + chmod -R u+w $out/nixos + if [ ! -e $out/nixos/nixpkgs ]; then + ln -s . $out/nixos/nixpkgs + fi + rm -rf $out/nixos/.git + echo -n ${config.system.nixos.versionSuffix} > $out/nixos/.version-suffix + ''; + + closureInfo = pkgs.closureInfo { + rootPaths = [ config.system.build.toplevel ] + ++ (lib.optional includeChannel channelSources); + }; + + modulesTree = pkgs.aggregateModules + (with config.boot.kernelPackages; [ kernel zfs ]); + + tools = lib.makeBinPath ( + with pkgs; [ + config.system.build.nixos-enter + config.system.build.nixos-install + dosfstools + e2fsprogs + gptfdisk + nix + parted + util-linux + zfs + ] + ); + + hasDefinedMount = disk: ((disk.mount or null) != null); + + stringifyProperties = prefix: properties: lib.concatStringsSep " \\\n" ( + lib.mapAttrsToList + ( + property: value: "${prefix} ${lib.escapeShellArg property}=${lib.escapeShellArg value}" + ) + properties + ); + + createDatasets = + let + datasetlist = lib.mapAttrsToList lib.nameValuePair datasets; + sorted = lib.sort (left: right: (lib.stringLength left.name) < (lib.stringLength right.name)) datasetlist; + cmd = { name, value }: + let + properties = stringifyProperties "-o" (value.properties or {}); + in + "zfs create -p ${properties} ${name}"; + in + lib.concatMapStringsSep "\n" cmd sorted; + + mountDatasets = + let + datasetlist = lib.mapAttrsToList lib.nameValuePair datasets; + mounts = lib.filter ({ value, ... }: hasDefinedMount value) datasetlist; + sorted = lib.sort (left: right: (lib.stringLength left.value.mount) < (lib.stringLength right.value.mount)) mounts; + cmd = { name, value }: + '' + mkdir -p /mnt${lib.escapeShellArg value.mount} + mount -t zfs ${name} /mnt${lib.escapeShellArg value.mount} + ''; + in + lib.concatMapStringsSep "\n" cmd sorted; + + unmountDatasets = + let + datasetlist = lib.mapAttrsToList lib.nameValuePair datasets; + mounts = lib.filter ({ value, ... }: hasDefinedMount value) datasetlist; + sorted = lib.sort (left: right: (lib.stringLength left.value.mount) > (lib.stringLength right.value.mount)) mounts; + cmd = { name, value }: + '' + umount /mnt${lib.escapeShellArg value.mount} + ''; + in + lib.concatMapStringsSep "\n" cmd sorted; + + + fileSystemsCfgFile = + let + mountable = lib.filterAttrs (_: value: hasDefinedMount value) datasets; + in + pkgs.runCommand "filesystem-config.nix" { + buildInputs = with pkgs; [ jq nixpkgs-fmt ]; + filesystems = builtins.toJSON { + fileSystems = lib.mapAttrs' + ( + dataset: attrs: + { + name = attrs.mount; + value = { + fsType = "zfs"; + device = "${dataset}"; + }; + } + ) + mountable; + }; + passAsFile = [ "filesystems" ]; + } '' + ( + echo "builtins.fromJSON '''" + jq . < "$filesystemsPath" + echo "'''" + ) > $out + + nixpkgs-fmt $out + ''; + + mergedConfig = + if configFile == null + then fileSystemsCfgFile + else + pkgs.runCommand "configuration.nix" { + buildInputs = with pkgs; [ nixpkgs-fmt ]; + } + '' + ( + echo '{ imports = [' + printf "(%s)\n" "$(cat ${fileSystemsCfgFile})"; + printf "(%s)\n" "$(cat ${configFile})"; + echo ']; }' + ) > $out + + nixpkgs-fmt $out + ''; + + image = ( + pkgs.vmTools.override { + rootModules = + [ "zfs" "9p" "9pnet_virtio" "virtio_pci" "virtio_blk" ] ++ + (pkgs.lib.optional pkgs.stdenv.hostPlatform.isx86 "rtc_cmos"); + kernel = modulesTree; + } + ).runInLinuxVM ( + pkgs.runCommand name + { + QEMU_OPTS = "-drive file=$bootDiskImage,if=virtio,cache=unsafe,werror=report" + + " -drive file=$rootDiskImage,if=virtio,cache=unsafe,werror=report"; + inherit memSize; + preVM = '' + PATH=$PATH:${pkgs.qemu_kvm}/bin + mkdir $out + bootDiskImage=boot.raw + qemu-img create -f raw $bootDiskImage ${toString bootSize}M + + rootDiskImage=root.raw + qemu-img create -f raw $rootDiskImage ${toString rootSize}M + ''; + + postVM = '' + ${if formatOpt == "raw" then '' + mv $bootDiskImage $out/${bootFilename} + mv $rootDiskImage $out/${rootFilename} + '' else '' + ${pkgs.qemu_kvm}/bin/qemu-img convert -f raw -O ${formatOpt} ${compress} $bootDiskImage $out/${bootFilename} + ${pkgs.qemu_kvm}/bin/qemu-img convert -f raw -O ${formatOpt} ${compress} $rootDiskImage $out/${rootFilename} + ''} + bootDiskImage=$out/${bootFilename} + rootDiskImage=$out/${rootFilename} + set -x + ${postVM} + ''; + } '' + export PATH=${tools}:$PATH + set -x + + cp -sv /dev/vda /dev/sda + cp -sv /dev/vda /dev/xvda + + parted --script /dev/vda -- \ + mklabel gpt \ + mkpart no-fs 1MiB 2MiB \ + set 1 bios_grub on \ + align-check optimal 1 \ + mkpart ESP fat32 2MiB -1MiB \ + align-check optimal 2 \ + print + + sfdisk --dump /dev/vda + + + zpool create \ + ${stringifyProperties " -o" rootPoolProperties} \ + ${stringifyProperties " -O" rootPoolFilesystemProperties} \ + ${rootPoolName} /dev/vdb + parted --script /dev/vdb -- print + + ${createDatasets} + ${mountDatasets} + + mkdir -p /mnt/boot + mkfs.vfat -n ESP /dev/vda2 + mount /dev/vda2 /mnt/boot + + mount + + # Install a configuration.nix + mkdir -p /mnt/etc/nixos + # `cat` so it is mutable on the fs + cat ${mergedConfig} > /mnt/etc/nixos/configuration.nix + + export NIX_STATE_DIR=$TMPDIR/state + nix-store --load-db < ${closureInfo}/registration + + nixos-install \ + --root /mnt \ + --no-root-passwd \ + --system ${config.system.build.toplevel} \ + --substituters "" \ + ${lib.optionalString includeChannel ''--channel ${channelSources}''} + + df -h + + umount /mnt/boot + ${unmountDatasets} + + zpool export ${rootPoolName} + '' + ); +in +image diff --git a/nixpkgs/nixos/lib/make-options-doc/default.nix b/nixpkgs/nixos/lib/make-options-doc/default.nix new file mode 100644 index 000000000000..284934a7608e --- /dev/null +++ b/nixpkgs/nixos/lib/make-options-doc/default.nix @@ -0,0 +1,175 @@ +/* Generate JSON, XML and DocBook documentation for given NixOS options. + + Minimal example: + + { pkgs, }: + + let + eval = import (pkgs.path + "/nixos/lib/eval-config.nix") { + baseModules = [ + ../module.nix + ]; + modules = []; + }; + in pkgs.nixosOptionsDoc { + options = eval.options; + } + +*/ +{ pkgs +, lib +, options +, transformOptions ? lib.id # function for additional transformations of the options +, documentType ? "appendix" # TODO deprecate "appendix" in favor of "none" + # and/or rename function to moduleOptionDoc for clean slate + + # If you include more than one option list into a document, you need to + # provide different ids. +, variablelistId ? "configuration-variable-list" + # String to prefix to the option XML/HTML id attributes. +, optionIdPrefix ? "opt-" +, revision ? "" # Specify revision for the options +# a set of options the docs we are generating will be merged into, as if by recursiveUpdate. +# used to split the options doc build into a static part (nixos/modules) and a dynamic part +# (non-nixos modules imported via configuration.nix, other module sources). +, baseOptionsJSON ? null +# instead of printing warnings for eg options with missing descriptions (which may be lost +# by nix build unless -L is given), emit errors instead and fail the build +, warningsAreErrors ? true +# allow docbook option docs if `true`. only markdown documentation is allowed when set to +# `false`, and a different renderer may be used with different bugs and performance +# characteristics but (hopefully) indistinguishable output. +# deprecated since 23.11. +# TODO remove in a while. +, allowDocBook ? false +# whether lib.mdDoc is required for descriptions to be read as markdown. +# deprecated since 23.11. +# TODO remove in a while. +, markdownByDefault ? true +}: + +assert markdownByDefault && ! allowDocBook; + +let + rawOpts = lib.optionAttrSetToDocList options; + transformedOpts = map transformOptions rawOpts; + filteredOpts = lib.filter (opt: opt.visible && !opt.internal) transformedOpts; + optionsList = lib.flip map filteredOpts + (opt: opt + // lib.optionalAttrs (opt ? relatedPackages && opt.relatedPackages != []) { relatedPackages = genRelatedPackages opt.relatedPackages opt.name; } + ); + + # Generate DocBook documentation for a list of packages. This is + # what `relatedPackages` option of `mkOption` from + # ../../../lib/options.nix influences. + # + # Each element of `relatedPackages` can be either + # - a string: that will be interpreted as an attribute name from `pkgs` and turned into a link + # to search.nixos.org, + # - a list: that will be interpreted as an attribute path from `pkgs` and turned into a link + # to search.nixos.org, + # - an attrset: that can specify `name`, `path`, `comment` + # (either of `name`, `path` is required, the rest are optional). + # + # NOTE: No checks against `pkgs` are made to ensure that the referenced package actually exists. + # Such checks are not compatible with option docs caching. + genRelatedPackages = packages: optName: + let + unpack = p: if lib.isString p then { name = p; } + else if lib.isList p then { path = p; } + else p; + describe = args: + let + title = args.title or null; + name = args.name or (lib.concatStringsSep "." args.path); + in '' + - [${lib.optionalString (title != null) "${title} aka "}`pkgs.${name}`]( + https://search.nixos.org/packages?show=${name}&sort=relevance&query=${name} + )${ + lib.optionalString (args ? comment) "\n\n ${args.comment}" + } + ''; + in lib.concatMapStrings (p: describe (unpack p)) packages; + + optionsNix = builtins.listToAttrs (map (o: { name = o.name; value = removeAttrs o ["name" "visible" "internal"]; }) optionsList); + +in rec { + inherit optionsNix; + + optionsAsciiDoc = pkgs.runCommand "options.adoc" { + nativeBuildInputs = [ pkgs.nixos-render-docs ]; + } '' + nixos-render-docs -j $NIX_BUILD_CORES options asciidoc \ + --manpage-urls ${pkgs.path + "/doc/manpage-urls.json"} \ + --revision ${lib.escapeShellArg revision} \ + ${optionsJSON}/share/doc/nixos/options.json \ + $out + ''; + + optionsCommonMark = pkgs.runCommand "options.md" { + nativeBuildInputs = [ pkgs.nixos-render-docs ]; + } '' + nixos-render-docs -j $NIX_BUILD_CORES options commonmark \ + --manpage-urls ${pkgs.path + "/doc/manpage-urls.json"} \ + --revision ${lib.escapeShellArg revision} \ + ${optionsJSON}/share/doc/nixos/options.json \ + $out + ''; + + optionsJSON = pkgs.runCommand "options.json" + { meta.description = "List of NixOS options in JSON format"; + nativeBuildInputs = [ + pkgs.brotli + pkgs.python3 + ]; + options = builtins.toFile "options.json" + (builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix)); + # merge with an empty set if baseOptionsJSON is null to run markdown + # processing on the input options + baseJSON = + if baseOptionsJSON == null + then builtins.toFile "base.json" "{}" + else baseOptionsJSON; + } + '' + # Export list of options in different format. + dst=$out/share/doc/nixos + mkdir -p $dst + + TOUCH_IF_DB=$dst/.used-docbook \ + python ${./mergeJSON.py} \ + ${lib.optionalString warningsAreErrors "--warnings-are-errors"} \ + $baseJSON $options \ + > $dst/options.json + + if grep /nixpkgs/nixos/modules $dst/options.json; then + echo "The manual appears to depend on the location of Nixpkgs, which is bad" + echo "since this prevents sharing via the NixOS channel. This is typically" + echo "caused by an option default that refers to a relative path (see above" + echo "for hints about the offending path)." + exit 1 + fi + + brotli -9 < $dst/options.json > $dst/options.json.br + + mkdir -p $out/nix-support + echo "file json $dst/options.json" >> $out/nix-support/hydra-build-products + echo "file json-br $dst/options.json.br" >> $out/nix-support/hydra-build-products + ''; + + optionsDocBook = lib.warn "optionsDocBook is deprecated since 23.11 and will be removed in 24.05" + (pkgs.runCommand "options-docbook.xml" { + nativeBuildInputs = [ + pkgs.nixos-render-docs + ]; + } '' + nixos-render-docs -j $NIX_BUILD_CORES options docbook \ + --manpage-urls ${pkgs.path + "/doc/manpage-urls.json"} \ + --revision ${lib.escapeShellArg revision} \ + --document-type ${lib.escapeShellArg documentType} \ + --varlist-id ${lib.escapeShellArg variablelistId} \ + --id-prefix ${lib.escapeShellArg optionIdPrefix} \ + ${optionsJSON}/share/doc/nixos/options.json \ + "$out" + ''); +} diff --git a/nixpkgs/nixos/lib/make-options-doc/mergeJSON.py b/nixpkgs/nixos/lib/make-options-doc/mergeJSON.py new file mode 100644 index 000000000000..4be83fcb827b --- /dev/null +++ b/nixpkgs/nixos/lib/make-options-doc/mergeJSON.py @@ -0,0 +1,104 @@ +import collections +import json +import os +import sys +from typing import Any, Dict, List + +JSON = Dict[str, Any] + +class Key: + def __init__(self, path: List[str]): + self.path = path + def __hash__(self): + result = 0 + for id in self.path: + result ^= hash(id) + return result + def __eq__(self, other): + return type(self) is type(other) and self.path == other.path + +Option = collections.namedtuple('Option', ['name', 'value']) + +# pivot a dict of options keyed by their display name to a dict keyed by their path +def pivot(options: Dict[str, JSON]) -> Dict[Key, Option]: + result: Dict[Key, Option] = dict() + for (name, opt) in options.items(): + result[Key(opt['loc'])] = Option(name, opt) + return result + +# pivot back to indexed-by-full-name +# like the docbook build we'll just fail if multiple options with differing locs +# render to the same option name. +def unpivot(options: Dict[Key, Option]) -> Dict[str, JSON]: + result: Dict[str, Dict] = dict() + for (key, opt) in options.items(): + if opt.name in result: + raise RuntimeError( + 'multiple options with colliding ids found', + opt.name, + result[opt.name]['loc'], + opt.value['loc'], + ) + result[opt.name] = opt.value + return result + +warningsAreErrors = False +optOffset = 0 +for arg in sys.argv[1:]: + if arg == "--warnings-are-errors": + optOffset += 1 + warningsAreErrors = True + +options = pivot(json.load(open(sys.argv[1 + optOffset], 'r'))) +overrides = pivot(json.load(open(sys.argv[2 + optOffset], 'r'))) + +# fix up declaration paths in lazy options, since we don't eval them from a full nixpkgs dir +for (k, v) in options.items(): + # The _module options are not declared in nixos/modules + if v.value['loc'][0] != "_module": + v.value['declarations'] = list(map(lambda s: f'nixos/modules/{s}' if isinstance(s, str) else s, v.value['declarations'])) + +# merge both descriptions +for (k, v) in overrides.items(): + cur = options.setdefault(k, v).value + for (ok, ov) in v.value.items(): + if ok == 'declarations': + decls = cur[ok] + for d in ov: + if d not in decls: + decls += [d] + elif ok == "type": + # ignore types of placeholder options + if ov != "_unspecified" or cur[ok] == "_unspecified": + cur[ok] = ov + elif ov is not None or cur.get(ok, None) is None: + cur[ok] = ov + +severity = "error" if warningsAreErrors else "warning" + +# check that every option has a description +hasWarnings = False +hasErrors = False +for (k, v) in options.items(): + if v.value.get('description', None) is None: + hasWarnings = True + print(f"\x1b[1;31m{severity}: option {v.name} has no description\x1b[0m", file=sys.stderr) + v.value['description'] = "This option has no description." + if v.value.get('type', "unspecified") == "unspecified": + hasWarnings = True + print( + f"\x1b[1;31m{severity}: option {v.name} has no type. Please specify a valid type, see " + + "https://nixos.org/manual/nixos/stable/index.html#sec-option-types\x1b[0m", file=sys.stderr) + +if hasErrors: + sys.exit(1) +if hasWarnings and warningsAreErrors: + print( + "\x1b[1;31m" + + "Treating warnings as errors. Set documentation.nixos.options.warningsAreErrors " + + "to false to ignore these warnings." + + "\x1b[0m", + file=sys.stderr) + sys.exit(1) + +json.dump(unpivot(options), fp=sys.stdout) diff --git a/nixpkgs/nixos/lib/make-single-disk-zfs-image.nix b/nixpkgs/nixos/lib/make-single-disk-zfs-image.nix new file mode 100644 index 000000000000..585fa93b7fa0 --- /dev/null +++ b/nixpkgs/nixos/lib/make-single-disk-zfs-image.nix @@ -0,0 +1,316 @@ +# Note: This is a private API, internal to NixOS. Its interface is subject +# to change without notice. +# +# The result of this builder is a single disk image, partitioned like this: +# +# * partition #1: a very small, 1MiB partition to leave room for Grub. +# +# * partition #2: boot, a partition formatted with FAT to be used for /boot. +# FAT is chosen to support EFI. +# +# * partition #3: nixos, a partition dedicated to a zpool. +# +# This single-disk approach does not satisfy ZFS's requirements for autoexpand, +# however automation can expand it anyway. For example, with +# `services.zfs.expandOnBoot`. +{ lib +, pkgs +, # The NixOS configuration to be installed onto the disk image. + config + +, # size of the FAT partition, in megabytes. + bootSize ? 1024 + + , # memory allocated for virtualized build instance + memSize ? 1024 + +, # The size of the root partition, in megabytes. + rootSize ? 2048 + +, # The name of the ZFS pool + rootPoolName ? "tank" + +, # zpool properties + rootPoolProperties ? { + autoexpand = "on"; + } +, # pool-wide filesystem properties + rootPoolFilesystemProperties ? { + acltype = "posixacl"; + atime = "off"; + compression = "on"; + mountpoint = "legacy"; + xattr = "sa"; + } + +, # datasets, with per-attribute options: + # mount: (optional) mount point in the VM + # properties: (optional) ZFS properties on the dataset, like filesystemProperties + # Notes: + # 1. datasets will be created from shorter to longer names as a simple topo-sort + # 2. you should define a root's dataset's mount for `/` + datasets ? { } + +, # The files and directories to be placed in the target file system. + # This is a list of attribute sets {source, target} where `source' + # is the file system object (regular file or directory) to be + # grafted in the file system at path `target'. + contents ? [ ] + +, # The initial NixOS configuration file to be copied to + # /etc/nixos/configuration.nix. This configuration will be embedded + # inside a configuration which includes the described ZFS fileSystems. + configFile ? null + +, # Shell code executed after the VM has finished. + postVM ? "" + +, name ? "nixos-disk-image" + +, # Disk image format, one of qcow2, qcow2-compressed, vdi, vpc, raw. + format ? "raw" + +, # Include a copy of Nixpkgs in the disk image + includeChannel ? true +}: +let + formatOpt = if format == "qcow2-compressed" then "qcow2" else format; + + compress = lib.optionalString (format == "qcow2-compressed") "-c"; + + filenameSuffix = "." + { + qcow2 = "qcow2"; + vdi = "vdi"; + vpc = "vhd"; + raw = "img"; + }.${formatOpt} or formatOpt; + rootFilename = "nixos.root${filenameSuffix}"; + + # FIXME: merge with channel.nix / make-channel.nix. + channelSources = + let + nixpkgs = lib.cleanSource pkgs.path; + in + pkgs.runCommand "nixos-${config.system.nixos.version}" { } '' + mkdir -p $out + cp -prd ${nixpkgs.outPath} $out/nixos + chmod -R u+w $out/nixos + if [ ! -e $out/nixos/nixpkgs ]; then + ln -s . $out/nixos/nixpkgs + fi + rm -rf $out/nixos/.git + echo -n ${config.system.nixos.versionSuffix} > $out/nixos/.version-suffix + ''; + + closureInfo = pkgs.closureInfo { + rootPaths = [ config.system.build.toplevel ] + ++ (lib.optional includeChannel channelSources); + }; + + modulesTree = pkgs.aggregateModules + (with config.boot.kernelPackages; [ kernel zfs ]); + + tools = lib.makeBinPath ( + with pkgs; [ + config.system.build.nixos-enter + config.system.build.nixos-install + dosfstools + e2fsprogs + gptfdisk + nix + parted + util-linux + zfs + ] + ); + + hasDefinedMount = disk: ((disk.mount or null) != null); + + stringifyProperties = prefix: properties: lib.concatStringsSep " \\\n" ( + lib.mapAttrsToList + ( + property: value: "${prefix} ${lib.escapeShellArg property}=${lib.escapeShellArg value}" + ) + properties + ); + + createDatasets = + let + datasetlist = lib.mapAttrsToList lib.nameValuePair datasets; + sorted = lib.sort (left: right: (lib.stringLength left.name) < (lib.stringLength right.name)) datasetlist; + cmd = { name, value }: + let + properties = stringifyProperties "-o" (value.properties or { }); + in + "zfs create -p ${properties} ${name}"; + in + lib.concatMapStringsSep "\n" cmd sorted; + + mountDatasets = + let + datasetlist = lib.mapAttrsToList lib.nameValuePair datasets; + mounts = lib.filter ({ value, ... }: hasDefinedMount value) datasetlist; + sorted = lib.sort (left: right: (lib.stringLength left.value.mount) < (lib.stringLength right.value.mount)) mounts; + cmd = { name, value }: + '' + mkdir -p /mnt${lib.escapeShellArg value.mount} + mount -t zfs ${name} /mnt${lib.escapeShellArg value.mount} + ''; + in + lib.concatMapStringsSep "\n" cmd sorted; + + unmountDatasets = + let + datasetlist = lib.mapAttrsToList lib.nameValuePair datasets; + mounts = lib.filter ({ value, ... }: hasDefinedMount value) datasetlist; + sorted = lib.sort (left: right: (lib.stringLength left.value.mount) > (lib.stringLength right.value.mount)) mounts; + cmd = { name, value }: + '' + umount /mnt${lib.escapeShellArg value.mount} + ''; + in + lib.concatMapStringsSep "\n" cmd sorted; + + + fileSystemsCfgFile = + let + mountable = lib.filterAttrs (_: value: hasDefinedMount value) datasets; + in + pkgs.runCommand "filesystem-config.nix" + { + buildInputs = with pkgs; [ jq nixpkgs-fmt ]; + filesystems = builtins.toJSON { + fileSystems = lib.mapAttrs' + ( + dataset: attrs: + { + name = attrs.mount; + value = { + fsType = "zfs"; + device = "${dataset}"; + }; + } + ) + mountable; + }; + passAsFile = [ "filesystems" ]; + } '' + ( + echo "builtins.fromJSON '''" + jq . < "$filesystemsPath" + echo "'''" + ) > $out + + nixpkgs-fmt $out + ''; + + mergedConfig = + if configFile == null + then fileSystemsCfgFile + else + pkgs.runCommand "configuration.nix" + { + buildInputs = with pkgs; [ nixpkgs-fmt ]; + } + '' + ( + echo '{ imports = [' + printf "(%s)\n" "$(cat ${fileSystemsCfgFile})"; + printf "(%s)\n" "$(cat ${configFile})"; + echo ']; }' + ) > $out + + nixpkgs-fmt $out + ''; + + image = ( + pkgs.vmTools.override { + rootModules = + [ "zfs" "9p" "9pnet_virtio" "virtio_pci" "virtio_blk" ] ++ + (pkgs.lib.optional pkgs.stdenv.hostPlatform.isx86 "rtc_cmos"); + kernel = modulesTree; + } + ).runInLinuxVM ( + pkgs.runCommand name + { + inherit memSize; + QEMU_OPTS = "-drive file=$rootDiskImage,if=virtio,cache=unsafe,werror=report"; + preVM = '' + PATH=$PATH:${pkgs.qemu_kvm}/bin + mkdir $out + + rootDiskImage=root.raw + qemu-img create -f raw $rootDiskImage ${toString (bootSize + rootSize)}M + ''; + + postVM = '' + ${if formatOpt == "raw" then '' + mv $rootDiskImage $out/${rootFilename} + '' else '' + ${pkgs.qemu_kvm}/bin/qemu-img convert -f raw -O ${formatOpt} ${compress} $rootDiskImage $out/${rootFilename} + ''} + rootDiskImage=$out/${rootFilename} + set -x + ${postVM} + ''; + } '' + export PATH=${tools}:$PATH + set -x + + cp -sv /dev/vda /dev/sda + cp -sv /dev/vda /dev/xvda + + parted --script /dev/vda -- \ + mklabel gpt \ + mkpart no-fs 1MiB 2MiB \ + set 1 bios_grub on \ + align-check optimal 1 \ + mkpart primary fat32 2MiB ${toString bootSize}MiB \ + align-check optimal 2 \ + mkpart primary fat32 ${toString bootSize}MiB -1MiB \ + align-check optimal 3 \ + print + + sfdisk --dump /dev/vda + + + zpool create \ + ${stringifyProperties " -o" rootPoolProperties} \ + ${stringifyProperties " -O" rootPoolFilesystemProperties} \ + ${rootPoolName} /dev/vda3 + parted --script /dev/vda -- print + + ${createDatasets} + ${mountDatasets} + + mkdir -p /mnt/boot + mkfs.vfat -n ESP /dev/vda2 + mount /dev/vda2 /mnt/boot + + mount + + # Install a configuration.nix + mkdir -p /mnt/etc/nixos + # `cat` so it is mutable on the fs + cat ${mergedConfig} > /mnt/etc/nixos/configuration.nix + + export NIX_STATE_DIR=$TMPDIR/state + nix-store --load-db < ${closureInfo}/registration + + nixos-install \ + --root /mnt \ + --no-root-passwd \ + --system ${config.system.build.toplevel} \ + --substituters "" \ + ${lib.optionalString includeChannel ''--channel ${channelSources}''} + + df -h + + umount /mnt/boot + ${unmountDatasets} + + zpool export ${rootPoolName} + '' + ); +in +image diff --git a/nixpkgs/nixos/lib/make-squashfs.nix b/nixpkgs/nixos/lib/make-squashfs.nix new file mode 100644 index 000000000000..f28e2c671580 --- /dev/null +++ b/nixpkgs/nixos/lib/make-squashfs.nix @@ -0,0 +1,46 @@ +{ lib, stdenv, squashfsTools, closureInfo + +, fileName ? "squashfs" +, # The root directory of the squashfs filesystem is filled with the + # closures of the Nix store paths listed here. + storeContents ? [] + # Pseudo files to be added to squashfs image +, pseudoFiles ? [] +, noStrip ? false +, # Compression parameters. + # For zstd compression you can use "zstd -Xcompression-level 6". + comp ? "xz -Xdict-size 100%" +}: + +let + pseudoFilesArgs = lib.concatMapStrings (f: ''-p "${f}" '') pseudoFiles; + compFlag = if comp == null then "-no-compression" else "-comp ${comp}"; +in +stdenv.mkDerivation { + name = "${fileName}.img"; + __structuredAttrs = true; + + nativeBuildInputs = [ squashfsTools ]; + + buildCommand = + '' + closureInfo=${closureInfo { rootPaths = storeContents; }} + + # Also include a manifest of the closures in a format suitable + # for nix-store --load-db. + cp $closureInfo/registration nix-path-registration + + '' + lib.optionalString stdenv.buildPlatform.is32bit '' + # 64 cores on i686 does not work + # fails with FATAL ERROR: mangle2:: xz compress failed with error code 5 + if ((NIX_BUILD_CORES > 48)); then + NIX_BUILD_CORES=48 + fi + '' + '' + + # Generate the squashfs image. + mksquashfs nix-path-registration $(cat $closureInfo/store-paths) $out ${pseudoFilesArgs} \ + -no-hardlinks ${lib.optionalString noStrip "-no-strip"} -keep-as-directory -all-root -b 1048576 ${compFlag} \ + -processors $NIX_BUILD_CORES + ''; +} diff --git a/nixpkgs/nixos/lib/make-system-tarball.nix b/nixpkgs/nixos/lib/make-system-tarball.nix new file mode 100644 index 000000000000..325792f97e8f --- /dev/null +++ b/nixpkgs/nixos/lib/make-system-tarball.nix @@ -0,0 +1,56 @@ +{ stdenv, closureInfo, pixz + +, # The file name of the resulting tarball + fileName ? "nixos-system-${stdenv.hostPlatform.system}" + +, # The files and directories to be placed in the tarball. + # This is a list of attribute sets {source, target} where `source' + # is the file system object (regular file or directory) to be + # grafted in the file system at path `target'. + contents + +, # In addition to `contents', the closure of the store paths listed + # in `packages' are also placed in the Nix store of the tarball. This is + # a list of attribute sets {object, symlink} where `object' if a + # store path whose closure will be copied, and `symlink' is a + # symlink to `object' that will be added to the tarball. + storeContents ? [] + + # Extra commands to be executed before archiving files +, extraCommands ? "" + + # Extra tar arguments +, extraArgs ? "" + # Command used for compression +, compressCommand ? "pixz -t" + # Extension for the compressed tarball +, compressionExtension ? ".xz" + # extra inputs, like the compressor to use +, extraInputs ? [ pixz ] +}: + +let + symlinks = map (x: x.symlink) storeContents; + objects = map (x: x.object) storeContents; +in + +stdenv.mkDerivation { + name = "tarball"; + builder = ./make-system-tarball.sh; + nativeBuildInputs = extraInputs; + + inherit fileName extraArgs extraCommands compressCommand; + + # !!! should use XML. + sources = map (x: x.source) contents; + targets = map (x: x.target) contents; + + # !!! should use XML. + inherit symlinks objects; + + closureInfo = closureInfo { + rootPaths = objects; + }; + + extension = compressionExtension; +} diff --git a/nixpkgs/nixos/lib/make-system-tarball.sh b/nixpkgs/nixos/lib/make-system-tarball.sh new file mode 100644 index 000000000000..1a0017a1799a --- /dev/null +++ b/nixpkgs/nixos/lib/make-system-tarball.sh @@ -0,0 +1,57 @@ +source $stdenv/setup + +sources_=($sources) +targets_=($targets) + +objects=($objects) +symlinks=($symlinks) + + +# Remove the initial slash from a path, since genisofs likes it that way. +stripSlash() { + res="$1" + if test "${res:0:1}" = /; then res=${res:1}; fi +} + +# Add the individual files. +for ((i = 0; i < ${#targets_[@]}; i++)); do + stripSlash "${targets_[$i]}" + mkdir -p "$(dirname "$res")" + cp -a "${sources_[$i]}" "$res" +done + + +# Add the closures of the top-level store objects. +chmod +w . +mkdir -p nix/store +for i in $(< $closureInfo/store-paths); do + cp -a "$i" "${i:1}" +done + + +# TODO tar ruxo +# Also include a manifest of the closures in a format suitable for +# nix-store --load-db. +cp $closureInfo/registration nix-path-registration + +# Add symlinks to the top-level store objects. +for ((n = 0; n < ${#objects[*]}; n++)); do + object=${objects[$n]} + symlink=${symlinks[$n]} + if test "$symlink" != "none"; then + mkdir -p $(dirname ./$symlink) + ln -s $object ./$symlink + fi +done + +$extraCommands + +mkdir -p $out/tarball + +rm env-vars + +time tar --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner -c * $extraArgs | $compressCommand > $out/tarball/$fileName.tar${extension} + +mkdir -p $out/nix-support +echo $system > $out/nix-support/system +echo "file system-tarball $out/tarball/$fileName.tar${extension}" > $out/nix-support/hydra-build-products diff --git a/nixpkgs/nixos/lib/qemu-common.nix b/nixpkgs/nixos/lib/qemu-common.nix new file mode 100644 index 000000000000..b946f62d93dc --- /dev/null +++ b/nixpkgs/nixos/lib/qemu-common.nix @@ -0,0 +1,65 @@ +# QEMU-related utilities shared between various Nix expressions. +{ lib, pkgs }: + +let + zeroPad = n: + lib.optionalString (n < 16) "0" + + (if n > 255 + then throw "Can't have more than 255 nets or nodes!" + else lib.toHexString n); +in + +rec { + qemuNicMac = net: machine: "52:54:00:12:${zeroPad net}:${zeroPad machine}"; + + qemuNICFlags = nic: net: machine: + [ + "-device virtio-net-pci,netdev=vlan${toString nic},mac=${qemuNicMac net machine}" + ''-netdev vde,id=vlan${toString nic},sock="$QEMU_VDE_SOCKET_${toString net}"'' + ]; + + qemuSerialDevice = + if with pkgs.stdenv.hostPlatform; isx86 || isLoongArch64 || isMips64 || isRiscV then "ttyS0" + else if (with pkgs.stdenv.hostPlatform; isAarch || isPower) then "ttyAMA0" + else throw "Unknown QEMU serial device for system '${pkgs.stdenv.hostPlatform.system}'"; + + qemuBinary = qemuPkg: + let + hostStdenv = qemuPkg.stdenv; + hostSystem = hostStdenv.system; + guestSystem = pkgs.stdenv.hostPlatform.system; + + linuxHostGuestMatrix = { + x86_64-linux = "${qemuPkg}/bin/qemu-kvm -cpu max"; + armv7l-linux = "${qemuPkg}/bin/qemu-system-arm -machine virt,accel=kvm:tcg -cpu max"; + aarch64-linux = "${qemuPkg}/bin/qemu-system-aarch64 -machine virt,gic-version=max,accel=kvm:tcg -cpu max"; + powerpc64le-linux = "${qemuPkg}/bin/qemu-system-ppc64 -machine powernv"; + powerpc64-linux = "${qemuPkg}/bin/qemu-system-ppc64 -machine powernv"; + x86_64-darwin = "${qemuPkg}/bin/qemu-kvm -cpu max"; + }; + otherHostGuestMatrix = { + aarch64-darwin = { + aarch64-linux = "${qemuPkg}/bin/qemu-system-aarch64 -machine virt,gic-version=2,accel=hvf:tcg -cpu max"; + inherit (otherHostGuestMatrix.x86_64-darwin) x86_64-linux; + }; + x86_64-darwin = { + x86_64-linux = "${qemuPkg}/bin/qemu-system-x86_64 -machine type=q35,accel=hvf:tcg -cpu max"; + }; + }; + + throwUnsupportedHostSystem = + let + supportedSystems = [ "linux" ] ++ (lib.attrNames otherHostGuestMatrix); + in + throw "Unsupported host system ${hostSystem}, supported: ${lib.concatStringsSep ", " supportedSystems}"; + throwUnsupportedGuestSystem = guestMap: + throw "Unsupported guest system ${guestSystem} for host ${hostSystem}, supported: ${lib.concatStringsSep ", " (lib.attrNames guestMap)}"; + in + if hostStdenv.isLinux then + linuxHostGuestMatrix.${guestSystem} or "${qemuPkg}/bin/qemu-kvm" + else + let + guestMap = (otherHostGuestMatrix.${hostSystem} or throwUnsupportedHostSystem); + in + (guestMap.${guestSystem} or (throwUnsupportedGuestSystem guestMap)); +} diff --git a/nixpkgs/nixos/lib/systemd-lib.nix b/nixpkgs/nixos/lib/systemd-lib.nix new file mode 100644 index 000000000000..347ee7303936 --- /dev/null +++ b/nixpkgs/nixos/lib/systemd-lib.nix @@ -0,0 +1,468 @@ +{ config, lib, pkgs }: + +with lib; + +let + cfg = config.systemd; + lndir = "${pkgs.buildPackages.xorg.lndir}/bin/lndir"; + systemd = cfg.package; +in rec { + + shellEscape = s: (replaceStrings [ "\\" ] [ "\\\\" ] s); + + mkPathSafeName = lib.replaceStrings ["@" ":" "\\" "[" "]"] ["-" "-" "-" "" ""]; + + # a type for options that take a unit name + unitNameType = types.strMatching "[a-zA-Z0-9@%:_.\\-]+[.](service|socket|device|mount|automount|swap|target|path|timer|scope|slice)"; + + makeUnit = name: unit: + if unit.enable then + pkgs.runCommand "unit-${mkPathSafeName name}" + { preferLocalBuild = true; + allowSubstitutes = false; + # unit.text can be null. But variables that are null listed in + # passAsFile are ignored by nix, resulting in no file being created, + # making the mv operation fail. + text = optionalString (unit.text != null) unit.text; + passAsFile = [ "text" ]; + } + '' + name=${shellEscape name} + mkdir -p "$out/$(dirname -- "$name")" + mv "$textPath" "$out/$name" + '' + else + pkgs.runCommand "unit-${mkPathSafeName name}-disabled" + { preferLocalBuild = true; + allowSubstitutes = false; + } + '' + name=${shellEscape name} + mkdir -p "$out/$(dirname "$name")" + ln -s /dev/null "$out/$name" + ''; + + boolValues = [true false "yes" "no"]; + + digits = map toString (range 0 9); + + isByteFormat = s: + let + l = reverseList (stringToCharacters s); + suffix = head l; + nums = tail l; + in elem suffix (["K" "M" "G" "T"] ++ digits) + && all (num: elem num digits) nums; + + assertByteFormat = name: group: attr: + optional (attr ? ${name} && ! isByteFormat attr.${name}) + "Systemd ${group} field `${name}' must be in byte format [0-9]+[KMGT]."; + + hexChars = stringToCharacters "0123456789abcdefABCDEF"; + + isMacAddress = s: stringLength s == 17 + && flip all (splitString ":" s) (bytes: + all (byte: elem byte hexChars) (stringToCharacters bytes) + ); + + assertMacAddress = name: group: attr: + optional (attr ? ${name} && ! isMacAddress attr.${name}) + "Systemd ${group} field `${name}' must be a valid MAC address."; + + assertNetdevMacAddress = name: group: attr: + optional (attr ? ${name} && (! isMacAddress attr.${name} && attr.${name} != "none")) + "Systemd ${group} field `${name}` must be a valid MAC address or the special value `none`."; + + + isPort = i: i >= 0 && i <= 65535; + + assertPort = name: group: attr: + optional (attr ? ${name} && ! isPort attr.${name}) + "Error on the systemd ${group} field `${name}': ${attr.name} is not a valid port number."; + + assertValueOneOf = name: values: group: attr: + optional (attr ? ${name} && !elem attr.${name} values) + "Systemd ${group} field `${name}' cannot have value `${toString attr.${name}}'."; + + assertValuesSomeOfOr = name: values: default: group: attr: + optional (attr ? ${name} && !(all (x: elem x values) (splitString " " attr.${name}) || attr.${name} == default)) + "Systemd ${group} field `${name}' cannot have value `${toString attr.${name}}'."; + + assertHasField = name: group: attr: + optional (!(attr ? ${name})) + "Systemd ${group} field `${name}' must exist."; + + assertRange = name: min: max: group: attr: + optional (attr ? ${name} && !(min <= attr.${name} && max >= attr.${name})) + "Systemd ${group} field `${name}' is outside the range [${toString min},${toString max}]"; + + assertMinimum = name: min: group: attr: + optional (attr ? ${name} && attr.${name} < min) + "Systemd ${group} field `${name}' must be greater than or equal to ${toString min}"; + + assertOnlyFields = fields: group: attr: + let badFields = filter (name: ! elem name fields) (attrNames attr); in + optional (badFields != [ ]) + "Systemd ${group} has extra fields [${concatStringsSep " " badFields}]."; + + assertInt = name: group: attr: + optional (attr ? ${name} && !isInt attr.${name}) + "Systemd ${group} field `${name}' is not an integer"; + + checkUnitConfig = group: checks: attrs: let + # We're applied at the top-level type (attrsOf unitOption), so the actual + # unit options might contain attributes from mkOverride and mkIf that we need to + # convert into single values before checking them. + defs = mapAttrs (const (v: + if v._type or "" == "override" then v.content + else if v._type or "" == "if" then v.content + else v + )) attrs; + errors = concatMap (c: c group defs) checks; + in if errors == [] then true + else builtins.trace (concatStringsSep "\n" errors) false; + + toOption = x: + if x == true then "true" + else if x == false then "false" + else toString x; + + attrsToSection = as: + concatStrings (concatLists (mapAttrsToList (name: value: + map (x: '' + ${name}=${toOption x} + '') + (if isList value then value else [value])) + as)); + + generateUnits = { allowCollisions ? true, type, units, upstreamUnits, upstreamWants, packages ? cfg.packages, package ? cfg.package }: + let + typeDir = ({ + system = "system"; + initrd = "system"; + user = "user"; + nspawn = "nspawn"; + }).${type}; + in pkgs.runCommand "${type}-units" + { preferLocalBuild = true; + allowSubstitutes = false; + } '' + mkdir -p $out + + # Copy the upstream systemd units we're interested in. + for i in ${toString upstreamUnits}; do + fn=${package}/example/systemd/${typeDir}/$i + if ! [ -e $fn ]; then echo "missing $fn"; false; fi + if [ -L $fn ]; then + target="$(readlink "$fn")" + if [ ''${target:0:3} = ../ ]; then + ln -s "$(readlink -f "$fn")" $out/ + else + cp -pd $fn $out/ + fi + else + ln -s $fn $out/ + fi + done + + # Copy .wants links, but only those that point to units that + # we're interested in. + for i in ${toString upstreamWants}; do + fn=${package}/example/systemd/${typeDir}/$i + if ! [ -e $fn ]; then echo "missing $fn"; false; fi + x=$out/$(basename $fn) + mkdir $x + for i in $fn/*; do + y=$x/$(basename $i) + cp -pd $i $y + if ! [ -e $y ]; then rm $y; fi + done + done + + # Symlink all units provided listed in systemd.packages. + packages="${toString packages}" + + # Filter duplicate directories + declare -A unique_packages + for k in $packages ; do unique_packages[$k]=1 ; done + + for i in ''${!unique_packages[@]}; do + for fn in $i/etc/systemd/${typeDir}/* $i/lib/systemd/${typeDir}/*; do + if ! [[ "$fn" =~ .wants$ ]]; then + if [[ -d "$fn" ]]; then + targetDir="$out/$(basename "$fn")" + mkdir -p "$targetDir" + ${lndir} "$fn" "$targetDir" + else + ln -s $fn $out/ + fi + fi + done + done + + # Symlink units defined by systemd.units where override strategy + # shall be automatically detected. If these are also provided by + # systemd or systemd.packages, then add them as + # <unit-name>.d/overrides.conf, which makes them extend the + # upstream unit. + for i in ${toString (mapAttrsToList + (n: v: v.unit) + (lib.filterAttrs (n: v: (attrByPath [ "overrideStrategy" ] "asDropinIfExists" v) == "asDropinIfExists") units))}; do + fn=$(basename $i/*) + if [ -e $out/$fn ]; then + if [ "$(readlink -f $i/$fn)" = /dev/null ]; then + ln -sfn /dev/null $out/$fn + else + ${if allowCollisions then '' + mkdir -p $out/$fn.d + ln -s $i/$fn $out/$fn.d/overrides.conf + '' else '' + echo "Found multiple derivations configuring $fn!" + exit 1 + ''} + fi + else + ln -fs $i/$fn $out/ + fi + done + + # Symlink units defined by systemd.units which shall be + # treated as drop-in file. + for i in ${toString (mapAttrsToList + (n: v: v.unit) + (lib.filterAttrs (n: v: v ? overrideStrategy && v.overrideStrategy == "asDropin") units))}; do + fn=$(basename $i/*) + mkdir -p $out/$fn.d + ln -s $i/$fn $out/$fn.d/overrides.conf + done + + # Create service aliases from aliases option. + ${concatStrings (mapAttrsToList (name: unit: + concatMapStrings (name2: '' + ln -sfn '${name}' $out/'${name2}' + '') (unit.aliases or [])) units)} + + # Create .wants and .requires symlinks from the wantedBy and + # requiredBy options. + ${concatStrings (mapAttrsToList (name: unit: + concatMapStrings (name2: '' + mkdir -p $out/'${name2}.wants' + ln -sfn '../${name}' $out/'${name2}.wants'/ + '') (unit.wantedBy or [])) units)} + + ${concatStrings (mapAttrsToList (name: unit: + concatMapStrings (name2: '' + mkdir -p $out/'${name2}.requires' + ln -sfn '../${name}' $out/'${name2}.requires'/ + '') (unit.requiredBy or [])) units)} + + ${optionalString (type == "system") '' + # Stupid misc. symlinks. + ln -s ${cfg.defaultUnit} $out/default.target + ln -s ${cfg.ctrlAltDelUnit} $out/ctrl-alt-del.target + ln -s rescue.target $out/kbrequest.target + + mkdir -p $out/getty.target.wants/ + ln -s ../autovt@tty1.service $out/getty.target.wants/ + + ln -s ../remote-fs.target $out/multi-user.target.wants/ + ''} + ''; # */ + + makeJobScript = name: text: + let + scriptName = replaceStrings [ "\\" "@" ] [ "-" "_" ] (shellEscape name); + out = (pkgs.writeShellScriptBin scriptName '' + set -e + ${text} + '').overrideAttrs (_: { + # The derivation name is different from the script file name + # to keep the script file name short to avoid cluttering logs. + name = "unit-script-${scriptName}"; + }); + in "${out}/bin/${scriptName}"; + + unitConfig = { config, name, options, ... }: { + config = { + unitConfig = + optionalAttrs (config.requires != []) + { Requires = toString config.requires; } + // optionalAttrs (config.wants != []) + { Wants = toString config.wants; } + // optionalAttrs (config.after != []) + { After = toString config.after; } + // optionalAttrs (config.before != []) + { Before = toString config.before; } + // optionalAttrs (config.bindsTo != []) + { BindsTo = toString config.bindsTo; } + // optionalAttrs (config.partOf != []) + { PartOf = toString config.partOf; } + // optionalAttrs (config.conflicts != []) + { Conflicts = toString config.conflicts; } + // optionalAttrs (config.requisite != []) + { Requisite = toString config.requisite; } + // optionalAttrs (config ? restartTriggers && config.restartTriggers != []) + { X-Restart-Triggers = "${pkgs.writeText "X-Restart-Triggers-${name}" (toString config.restartTriggers)}"; } + // optionalAttrs (config ? reloadTriggers && config.reloadTriggers != []) + { X-Reload-Triggers = "${pkgs.writeText "X-Reload-Triggers-${name}" (toString config.reloadTriggers)}"; } + // optionalAttrs (config.description != "") { + Description = config.description; } + // optionalAttrs (config.documentation != []) { + Documentation = toString config.documentation; } + // optionalAttrs (config.onFailure != []) { + OnFailure = toString config.onFailure; } + // optionalAttrs (config.onSuccess != []) { + OnSuccess = toString config.onSuccess; } + // optionalAttrs (options.startLimitIntervalSec.isDefined) { + StartLimitIntervalSec = toString config.startLimitIntervalSec; + } // optionalAttrs (options.startLimitBurst.isDefined) { + StartLimitBurst = toString config.startLimitBurst; + }; + }; + }; + + serviceConfig = { config, ... }: { + config.environment.PATH = mkIf (config.path != []) "${makeBinPath config.path}:${makeSearchPathOutput "bin" "sbin" config.path}"; + }; + + stage2ServiceConfig = { + imports = [ serviceConfig ]; + # Default path for systemd services. Should be quite minimal. + config.path = mkAfter [ + pkgs.coreutils + pkgs.findutils + pkgs.gnugrep + pkgs.gnused + systemd + ]; + }; + + stage1ServiceConfig = serviceConfig; + + mountConfig = { config, ... }: { + config = { + mountConfig = + { What = config.what; + Where = config.where; + } // optionalAttrs (config.type != "") { + Type = config.type; + } // optionalAttrs (config.options != "") { + Options = config.options; + }; + }; + }; + + automountConfig = { config, ... }: { + config = { + automountConfig = + { Where = config.where; + }; + }; + }; + + commonUnitText = def: lines: '' + [Unit] + ${attrsToSection def.unitConfig} + '' + lines + lib.optionalString (def.wantedBy != [ ]) '' + + [Install] + WantedBy=${concatStringsSep " " def.wantedBy} + ''; + + targetToUnit = name: def: + { inherit (def) aliases wantedBy requiredBy enable overrideStrategy; + text = + '' + [Unit] + ${attrsToSection def.unitConfig} + ''; + }; + + serviceToUnit = name: def: + { inherit (def) aliases wantedBy requiredBy enable overrideStrategy; + text = commonUnitText def ('' + [Service] + '' + (let env = cfg.globalEnvironment // def.environment; + in concatMapStrings (n: + let s = optionalString (env.${n} != null) + "Environment=${builtins.toJSON "${n}=${env.${n}}"}\n"; + # systemd max line length is now 1MiB + # https://github.com/systemd/systemd/commit/e6dde451a51dc5aaa7f4d98d39b8fe735f73d2af + in if stringLength s >= 1048576 then throw "The value of the environment variable ‘${n}’ in systemd service ‘${name}.service’ is too long." else s) (attrNames env)) + + (if def ? reloadIfChanged && def.reloadIfChanged then '' + X-ReloadIfChanged=true + '' else if (def ? restartIfChanged && !def.restartIfChanged) then '' + X-RestartIfChanged=false + '' else "") + + optionalString (def ? stopIfChanged && !def.stopIfChanged) '' + X-StopIfChanged=false + '' + attrsToSection def.serviceConfig); + }; + + socketToUnit = name: def: + { inherit (def) aliases wantedBy requiredBy enable overrideStrategy; + text = commonUnitText def '' + [Socket] + ${attrsToSection def.socketConfig} + ${concatStringsSep "\n" (map (s: "ListenStream=${s}") def.listenStreams)} + ${concatStringsSep "\n" (map (s: "ListenDatagram=${s}") def.listenDatagrams)} + ''; + }; + + timerToUnit = name: def: + { inherit (def) aliases wantedBy requiredBy enable overrideStrategy; + text = commonUnitText def '' + [Timer] + ${attrsToSection def.timerConfig} + ''; + }; + + pathToUnit = name: def: + { inherit (def) aliases wantedBy requiredBy enable overrideStrategy; + text = commonUnitText def '' + [Path] + ${attrsToSection def.pathConfig} + ''; + }; + + mountToUnit = name: def: + { inherit (def) aliases wantedBy requiredBy enable overrideStrategy; + text = commonUnitText def '' + [Mount] + ${attrsToSection def.mountConfig} + ''; + }; + + automountToUnit = name: def: + { inherit (def) aliases wantedBy requiredBy enable overrideStrategy; + text = commonUnitText def '' + [Automount] + ${attrsToSection def.automountConfig} + ''; + }; + + sliceToUnit = name: def: + { inherit (def) aliases wantedBy requiredBy enable overrideStrategy; + text = commonUnitText def '' + [Slice] + ${attrsToSection def.sliceConfig} + ''; + }; + + # Create a directory that contains systemd definition files from an attrset + # that contains the file names as keys and the content as values. The values + # in that attrset are determined by the supplied format. + definitions = directoryName: format: definitionAttrs: + let + listOfDefinitions = lib.mapAttrsToList + (name: format.generate "${name}.conf") + definitionAttrs; + in + pkgs.runCommand directoryName { } '' + mkdir -p $out + ${(lib.concatStringsSep "\n" + (map (pkg: "cp ${pkg} $out/${pkg.name}") listOfDefinitions) + )} + ''; + +} diff --git a/nixpkgs/nixos/lib/systemd-network-units.nix b/nixpkgs/nixos/lib/systemd-network-units.nix new file mode 100644 index 000000000000..1d5f823f3678 --- /dev/null +++ b/nixpkgs/nixos/lib/systemd-network-units.nix @@ -0,0 +1,249 @@ +{ lib, systemdUtils }: + +with lib; + +let + attrsToSection = systemdUtils.lib.attrsToSection; + commonMatchText = def: + optionalString (def.matchConfig != { }) '' + [Match] + ${attrsToSection def.matchConfig} + ''; +in { + linkToUnit = def: + commonMatchText def + '' + [Link] + ${attrsToSection def.linkConfig} + '' + def.extraConfig; + + netdevToUnit = def: + commonMatchText def + '' + [NetDev] + ${attrsToSection def.netdevConfig} + '' + optionalString (def.vlanConfig != { }) '' + [VLAN] + ${attrsToSection def.vlanConfig} + '' + optionalString (def.ipvlanConfig != { }) '' + [IPVLAN] + ${attrsToSection def.ipvlanConfig} + '' + optionalString (def.ipvtapConfig != { }) '' + [IPVTAP] + ${attrsToSection def.ipvtapConfig} + '' + optionalString (def.macvlanConfig != { }) '' + [MACVLAN] + ${attrsToSection def.macvlanConfig} + '' + optionalString (def.vxlanConfig != { }) '' + [VXLAN] + ${attrsToSection def.vxlanConfig} + '' + optionalString (def.tunnelConfig != { }) '' + [Tunnel] + ${attrsToSection def.tunnelConfig} + '' + optionalString (def.fooOverUDPConfig != { }) '' + [FooOverUDP] + ${attrsToSection def.fooOverUDPConfig} + '' + optionalString (def.peerConfig != { }) '' + [Peer] + ${attrsToSection def.peerConfig} + '' + optionalString (def.tunConfig != { }) '' + [Tun] + ${attrsToSection def.tunConfig} + '' + optionalString (def.tapConfig != { }) '' + [Tap] + ${attrsToSection def.tapConfig} + '' + optionalString (def.l2tpConfig != { }) '' + [L2TP] + ${attrsToSection def.l2tpConfig} + '' + flip concatMapStrings def.l2tpSessions (x: '' + [L2TPSession] + ${attrsToSection x.l2tpSessionConfig} + '') + optionalString (def.wireguardConfig != { }) '' + [WireGuard] + ${attrsToSection def.wireguardConfig} + '' + flip concatMapStrings def.wireguardPeers (x: '' + [WireGuardPeer] + ${attrsToSection x.wireguardPeerConfig} + '') + optionalString (def.bondConfig != { }) '' + [Bond] + ${attrsToSection def.bondConfig} + '' + optionalString (def.xfrmConfig != { }) '' + [Xfrm] + ${attrsToSection def.xfrmConfig} + '' + optionalString (def.vrfConfig != { }) '' + [VRF] + ${attrsToSection def.vrfConfig} + '' + optionalString (def.wlanConfig != { }) '' + [WLAN] + ${attrsToSection def.wlanConfig} + '' + optionalString (def.batmanAdvancedConfig != { }) '' + [BatmanAdvanced] + ${attrsToSection def.batmanAdvancedConfig} + '' + def.extraConfig; + + networkToUnit = def: + commonMatchText def + optionalString (def.linkConfig != { }) '' + [Link] + ${attrsToSection def.linkConfig} + '' + '' + [Network] + '' + attrsToSection def.networkConfig + + optionalString (def.address != [ ]) '' + ${concatStringsSep "\n" (map (s: "Address=${s}") def.address)} + '' + optionalString (def.gateway != [ ]) '' + ${concatStringsSep "\n" (map (s: "Gateway=${s}") def.gateway)} + '' + optionalString (def.dns != [ ]) '' + ${concatStringsSep "\n" (map (s: "DNS=${s}") def.dns)} + '' + optionalString (def.ntp != [ ]) '' + ${concatStringsSep "\n" (map (s: "NTP=${s}") def.ntp)} + '' + optionalString (def.bridge != [ ]) '' + ${concatStringsSep "\n" (map (s: "Bridge=${s}") def.bridge)} + '' + optionalString (def.bond != [ ]) '' + ${concatStringsSep "\n" (map (s: "Bond=${s}") def.bond)} + '' + optionalString (def.vrf != [ ]) '' + ${concatStringsSep "\n" (map (s: "VRF=${s}") def.vrf)} + '' + optionalString (def.vlan != [ ]) '' + ${concatStringsSep "\n" (map (s: "VLAN=${s}") def.vlan)} + '' + optionalString (def.macvlan != [ ]) '' + ${concatStringsSep "\n" (map (s: "MACVLAN=${s}") def.macvlan)} + '' + optionalString (def.macvtap != [ ]) '' + ${concatStringsSep "\n" (map (s: "MACVTAP=${s}") def.macvtap)} + '' + optionalString (def.vxlan != [ ]) '' + ${concatStringsSep "\n" (map (s: "VXLAN=${s}") def.vxlan)} + '' + optionalString (def.tunnel != [ ]) '' + ${concatStringsSep "\n" (map (s: "Tunnel=${s}") def.tunnel)} + '' + optionalString (def.xfrm != [ ]) '' + ${concatStringsSep "\n" (map (s: "Xfrm=${s}") def.xfrm)} + '' + "\n" + flip concatMapStrings def.addresses (x: '' + [Address] + ${attrsToSection x.addressConfig} + '') + flip concatMapStrings def.routingPolicyRules (x: '' + [RoutingPolicyRule] + ${attrsToSection x.routingPolicyRuleConfig} + '') + flip concatMapStrings def.routes (x: '' + [Route] + ${attrsToSection x.routeConfig} + '') + optionalString (def.dhcpV4Config != { }) '' + [DHCPv4] + ${attrsToSection def.dhcpV4Config} + '' + optionalString (def.dhcpV6Config != { }) '' + [DHCPv6] + ${attrsToSection def.dhcpV6Config} + '' + optionalString (def.dhcpPrefixDelegationConfig != { }) '' + [DHCPPrefixDelegation] + ${attrsToSection def.dhcpPrefixDelegationConfig} + '' + optionalString (def.ipv6AcceptRAConfig != { }) '' + [IPv6AcceptRA] + ${attrsToSection def.ipv6AcceptRAConfig} + '' + optionalString (def.dhcpServerConfig != { }) '' + [DHCPServer] + ${attrsToSection def.dhcpServerConfig} + '' + optionalString (def.ipv6SendRAConfig != { }) '' + [IPv6SendRA] + ${attrsToSection def.ipv6SendRAConfig} + '' + flip concatMapStrings def.ipv6Prefixes (x: '' + [IPv6Prefix] + ${attrsToSection x.ipv6PrefixConfig} + '') + flip concatMapStrings def.ipv6RoutePrefixes (x: '' + [IPv6RoutePrefix] + ${attrsToSection x.ipv6RoutePrefixConfig} + '') + flip concatMapStrings def.dhcpServerStaticLeases (x: '' + [DHCPServerStaticLease] + ${attrsToSection x.dhcpServerStaticLeaseConfig} + '') + optionalString (def.bridgeConfig != { }) '' + [Bridge] + ${attrsToSection def.bridgeConfig} + '' + flip concatMapStrings def.bridgeFDBs (x: '' + [BridgeFDB] + ${attrsToSection x.bridgeFDBConfig} + '') + flip concatMapStrings def.bridgeMDBs (x: '' + [BridgeMDB] + ${attrsToSection x.bridgeMDBConfig} + '') + optionalString (def.lldpConfig != { }) '' + [LLDP] + ${attrsToSection def.lldpConfig} + '' + optionalString (def.canConfig != { }) '' + [CAN] + ${attrsToSection def.canConfig} + '' + optionalString (def.ipoIBConfig != { }) '' + [IPoIB] + ${attrsToSection def.ipoIBConfig} + '' + optionalString (def.qdiscConfig != { }) '' + [QDisc] + ${attrsToSection def.qdiscConfig} + '' + optionalString (def.networkEmulatorConfig != { }) '' + [NetworkEmulator] + ${attrsToSection def.networkEmulatorConfig} + '' + optionalString (def.tokenBucketFilterConfig != { }) '' + [TokenBucketFilter] + ${attrsToSection def.tokenBucketFilterConfig} + '' + optionalString (def.pieConfig != { }) '' + [PIE] + ${attrsToSection def.pieConfig} + '' + optionalString (def.flowQueuePIEConfig != { }) '' + [FlowQueuePIE] + ${attrsToSection def.flowQueuePIEConfig} + '' + optionalString (def.stochasticFairBlueConfig != { }) '' + [StochasticFairBlue] + ${attrsToSection def.stochasticFairBlueConfig} + '' + optionalString (def.stochasticFairnessQueueingConfig != { }) '' + [StochasticFairnessQueueing] + ${attrsToSection def.stochasticFairnessQueueingConfig} + '' + optionalString (def.bfifoConfig != { }) '' + [BFIFO] + ${attrsToSection def.bfifoConfig} + '' + optionalString (def.pfifoConfig != { }) '' + [PFIFO] + ${attrsToSection def.pfifoConfig} + '' + optionalString (def.pfifoHeadDropConfig != { }) '' + [PFIFOHeadDrop] + ${attrsToSection def.pfifoHeadDropConfig} + '' + optionalString (def.pfifoFastConfig != { }) '' + [PFIFOFast] + ${attrsToSection def.pfifoFastConfig} + '' + optionalString (def.cakeConfig != { }) '' + [CAKE] + ${attrsToSection def.cakeConfig} + '' + optionalString (def.controlledDelayConfig != { }) '' + [ControlledDelay] + ${attrsToSection def.controlledDelayConfig} + '' + optionalString (def.deficitRoundRobinSchedulerConfig != { }) '' + [DeficitRoundRobinScheduler] + ${attrsToSection def.deficitRoundRobinSchedulerConfig} + '' + optionalString (def.deficitRoundRobinSchedulerClassConfig != { }) '' + [DeficitRoundRobinSchedulerClass] + ${attrsToSection def.deficitRoundRobinSchedulerClassConfig} + '' + optionalString (def.enhancedTransmissionSelectionConfig != { }) '' + [EnhancedTransmissionSelection] + ${attrsToSection def.enhancedTransmissionSelectionConfig} + '' + optionalString (def.genericRandomEarlyDetectionConfig != { }) '' + [GenericRandomEarlyDetection] + ${attrsToSection def.genericRandomEarlyDetectionConfig} + '' + optionalString (def.fairQueueingControlledDelayConfig != { }) '' + [FairQueueingControlledDelay] + ${attrsToSection def.fairQueueingControlledDelayConfig} + '' + optionalString (def.fairQueueingConfig != { }) '' + [FairQueueing] + ${attrsToSection def.fairQueueingConfig} + '' + optionalString (def.trivialLinkEqualizerConfig != { }) '' + [TrivialLinkEqualizer] + ${attrsToSection def.trivialLinkEqualizerConfig} + '' + optionalString (def.hierarchyTokenBucketConfig != { }) '' + [HierarchyTokenBucket] + ${attrsToSection def.hierarchyTokenBucketConfig} + '' + optionalString (def.hierarchyTokenBucketClassConfig != { }) '' + [HierarchyTokenBucketClass] + ${attrsToSection def.hierarchyTokenBucketClassConfig} + '' + optionalString (def.heavyHitterFilterConfig != { }) '' + [HeavyHitterFilter] + ${attrsToSection def.heavyHitterFilterConfig} + '' + optionalString (def.quickFairQueueingConfig != { }) '' + [QuickFairQueueing] + ${attrsToSection def.quickFairQueueingConfig} + '' + optionalString (def.quickFairQueueingConfigClass != { }) '' + [QuickFairQueueingClass] + ${attrsToSection def.quickFairQueueingConfigClass} + '' + flip concatMapStrings def.bridgeVLANs (x: '' + [BridgeVLAN] + ${attrsToSection x.bridgeVLANConfig} + '') + def.extraConfig; + +} diff --git a/nixpkgs/nixos/lib/systemd-types.nix b/nixpkgs/nixos/lib/systemd-types.nix new file mode 100644 index 000000000000..a109f248b170 --- /dev/null +++ b/nixpkgs/nixos/lib/systemd-types.nix @@ -0,0 +1,69 @@ +{ lib, systemdUtils, pkgs }: + +with systemdUtils.lib; +with systemdUtils.unitOptions; +with lib; + +rec { + units = with types; + attrsOf (submodule ({ name, config, ... }: { + options = concreteUnitOptions; + config = { unit = mkDefault (systemdUtils.lib.makeUnit name config); }; + })); + + services = with types; attrsOf (submodule [ stage2ServiceOptions unitConfig stage2ServiceConfig ]); + initrdServices = with types; attrsOf (submodule [ stage1ServiceOptions unitConfig stage1ServiceConfig ]); + + targets = with types; attrsOf (submodule [ stage2CommonUnitOptions unitConfig ]); + initrdTargets = with types; attrsOf (submodule [ stage1CommonUnitOptions unitConfig ]); + + sockets = with types; attrsOf (submodule [ stage2SocketOptions unitConfig ]); + initrdSockets = with types; attrsOf (submodule [ stage1SocketOptions unitConfig ]); + + timers = with types; attrsOf (submodule [ stage2TimerOptions unitConfig ]); + initrdTimers = with types; attrsOf (submodule [ stage1TimerOptions unitConfig ]); + + paths = with types; attrsOf (submodule [ stage2PathOptions unitConfig ]); + initrdPaths = with types; attrsOf (submodule [ stage1PathOptions unitConfig ]); + + slices = with types; attrsOf (submodule [ stage2SliceOptions unitConfig ]); + initrdSlices = with types; attrsOf (submodule [ stage1SliceOptions unitConfig ]); + + mounts = with types; listOf (submodule [ stage2MountOptions unitConfig mountConfig ]); + initrdMounts = with types; listOf (submodule [ stage1MountOptions unitConfig mountConfig ]); + + automounts = with types; listOf (submodule [ stage2AutomountOptions unitConfig automountConfig ]); + initrdAutomounts = with types; attrsOf (submodule [ stage1AutomountOptions unitConfig automountConfig ]); + + initrdContents = types.attrsOf (types.submodule ({ config, options, name, ... }: { + options = { + enable = mkEnableOption (lib.mdDoc "copying of this file and symlinking it") // { default = true; }; + + target = mkOption { + type = types.path; + description = lib.mdDoc '' + Path of the symlink. + ''; + default = name; + }; + + text = mkOption { + default = null; + type = types.nullOr types.lines; + description = lib.mdDoc "Text of the file."; + }; + + source = mkOption { + type = types.path; + description = lib.mdDoc "Path of the source file."; + }; + }; + + config = { + source = mkIf (config.text != null) ( + let name' = "initrd-" + baseNameOf name; + in mkDerivedConfig options.text (pkgs.writeText name') + ); + }; + })); +} diff --git a/nixpkgs/nixos/lib/systemd-unit-options.nix b/nixpkgs/nixos/lib/systemd-unit-options.nix new file mode 100644 index 000000000000..9c69bda471bb --- /dev/null +++ b/nixpkgs/nixos/lib/systemd-unit-options.nix @@ -0,0 +1,735 @@ +{ lib, systemdUtils }: + +with systemdUtils.lib; +with lib; + +let + checkService = checkUnitConfig "Service" [ + (assertValueOneOf "Type" [ + "exec" "simple" "forking" "oneshot" "dbus" "notify" "idle" + ]) + (assertValueOneOf "Restart" [ + "no" "on-success" "on-failure" "on-abnormal" "on-abort" "always" + ]) + ]; + +in rec { + + unitOption = mkOptionType { + name = "systemd option"; + merge = loc: defs: + let + defs' = filterOverrides defs; + in + if isList (head defs').value + then concatMap (def: + if builtins.typeOf def.value == "list" + then def.value + else + throw "The definitions for systemd unit options should be either all lists, representing repeatable options, or all non-lists, but for the option ${showOption loc}, the definitions are a mix of list and non-list ${lib.options.showDefs defs'}" + ) defs' + + else mergeEqualOption loc defs'; + }; + + sharedOptions = { + + enable = mkOption { + default = true; + type = types.bool; + description = lib.mdDoc '' + If set to false, this unit will be a symlink to + /dev/null. This is primarily useful to prevent specific + template instances + (e.g. `serial-getty@ttyS0`) from being + started. Note that `enable=true` does not + make a unit start by default at boot; if you want that, see + `wantedBy`. + ''; + }; + + overrideStrategy = mkOption { + default = "asDropinIfExists"; + type = types.enum [ "asDropinIfExists" "asDropin" ]; + description = lib.mdDoc '' + Defines how unit configuration is provided for systemd: + + `asDropinIfExists` creates a unit file when no unit file is provided by the package + otherwise a drop-in file name `overrides.conf`. + + `asDropin` creates a drop-in file named `overrides.conf`. + Mainly needed to define instances for systemd template units (e.g. `systemd-nspawn@mycontainer.service`). + + See also {manpage}`systemd.unit(5)`. + ''; + }; + + requiredBy = mkOption { + default = []; + type = types.listOf unitNameType; + description = lib.mdDoc '' + Units that require (i.e. depend on and need to go down with) this unit. + As discussed in the `wantedBy` option description this also creates + `.requires` symlinks automatically. + ''; + }; + + wantedBy = mkOption { + default = []; + type = types.listOf unitNameType; + description = lib.mdDoc '' + Units that want (i.e. depend on) this unit. The default method for + starting a unit by default at boot time is to set this option to + `["multi-user.target"]` for system services. Likewise for user units + (`systemd.user.<name>.*`) set it to `["default.target"]` to make a unit + start by default when the user `<name>` logs on. + + This option creates a `.wants` symlink in the given target that exists + statelessly without the need for running `systemctl enable`. + The `[Install]` section described in {manpage}`systemd.unit(5)` however is + not supported because it is a stateful process that does not fit well + into the NixOS design. + ''; + }; + + aliases = mkOption { + default = []; + type = types.listOf unitNameType; + description = lib.mdDoc "Aliases of that unit."; + }; + + }; + + concreteUnitOptions = sharedOptions // { + + text = mkOption { + type = types.nullOr types.str; + default = null; + description = lib.mdDoc "Text of this systemd unit."; + }; + + unit = mkOption { + internal = true; + description = lib.mdDoc "The generated unit."; + }; + + }; + + commonUnitOptions = { + options = sharedOptions // { + + description = mkOption { + default = ""; + type = types.singleLineStr; + description = lib.mdDoc "Description of this unit used in systemd messages and progress indicators."; + }; + + documentation = mkOption { + default = []; + type = types.listOf types.str; + description = lib.mdDoc "A list of URIs referencing documentation for this unit or its configuration."; + }; + + requires = mkOption { + default = []; + type = types.listOf unitNameType; + description = lib.mdDoc '' + Start the specified units when this unit is started, and stop + this unit when the specified units are stopped or fail. + ''; + }; + + wants = mkOption { + default = []; + type = types.listOf unitNameType; + description = lib.mdDoc '' + Start the specified units when this unit is started. + ''; + }; + + after = mkOption { + default = []; + type = types.listOf unitNameType; + description = lib.mdDoc '' + If the specified units are started at the same time as + this unit, delay this unit until they have started. + ''; + }; + + before = mkOption { + default = []; + type = types.listOf unitNameType; + description = lib.mdDoc '' + If the specified units are started at the same time as + this unit, delay them until this unit has started. + ''; + }; + + bindsTo = mkOption { + default = []; + type = types.listOf unitNameType; + description = lib.mdDoc '' + Like ‘requires’, but in addition, if the specified units + unexpectedly disappear, this unit will be stopped as well. + ''; + }; + + partOf = mkOption { + default = []; + type = types.listOf unitNameType; + description = lib.mdDoc '' + If the specified units are stopped or restarted, then this + unit is stopped or restarted as well. + ''; + }; + + conflicts = mkOption { + default = []; + type = types.listOf unitNameType; + description = lib.mdDoc '' + If the specified units are started, then this unit is stopped + and vice versa. + ''; + }; + + requisite = mkOption { + default = []; + type = types.listOf unitNameType; + description = lib.mdDoc '' + Similar to requires. However if the units listed are not started, + they will not be started and the transaction will fail. + ''; + }; + + unitConfig = mkOption { + default = {}; + example = { RequiresMountsFor = "/data"; }; + type = types.attrsOf unitOption; + description = lib.mdDoc '' + Each attribute in this set specifies an option in the + `[Unit]` section of the unit. See + {manpage}`systemd.unit(5)` for details. + ''; + }; + + onFailure = mkOption { + default = []; + type = types.listOf unitNameType; + description = lib.mdDoc '' + A list of one or more units that are activated when + this unit enters the "failed" state. + ''; + }; + + onSuccess = mkOption { + default = []; + type = types.listOf unitNameType; + description = lib.mdDoc '' + A list of one or more units that are activated when + this unit enters the "inactive" state. + ''; + }; + + startLimitBurst = mkOption { + type = types.int; + description = lib.mdDoc '' + Configure unit start rate limiting. Units which are started + more than startLimitBurst times within an interval time + interval are not permitted to start any more. + ''; + }; + + startLimitIntervalSec = mkOption { + type = types.int; + description = lib.mdDoc '' + Configure unit start rate limiting. Units which are started + more than startLimitBurst times within an interval time + interval are not permitted to start any more. + ''; + }; + + }; + }; + + stage2CommonUnitOptions = { + imports = [ + commonUnitOptions + ]; + + options = { + restartTriggers = mkOption { + default = []; + type = types.listOf types.unspecified; + description = lib.mdDoc '' + An arbitrary list of items such as derivations. If any item + in the list changes between reconfigurations, the service will + be restarted. + ''; + }; + + reloadTriggers = mkOption { + default = []; + type = types.listOf unitOption; + description = lib.mdDoc '' + An arbitrary list of items such as derivations. If any item + in the list changes between reconfigurations, the service will + be reloaded. If anything but a reload trigger changes in the + unit file, the unit will be restarted instead. + ''; + }; + }; + }; + stage1CommonUnitOptions = commonUnitOptions; + + serviceOptions = { name, config, ... }: { + options = { + + environment = mkOption { + default = {}; + type = with types; attrsOf (nullOr (oneOf [ str path package ])); + example = { PATH = "/foo/bar/bin"; LANG = "nl_NL.UTF-8"; }; + description = lib.mdDoc "Environment variables passed to the service's processes."; + }; + + path = mkOption { + default = []; + type = with types; listOf (oneOf [ package str ]); + description = lib.mdDoc '' + Packages added to the service's {env}`PATH` + environment variable. Both the {file}`bin` + and {file}`sbin` subdirectories of each + package are added. + ''; + }; + + serviceConfig = mkOption { + default = {}; + example = + { RestartSec = 5; + }; + type = types.addCheck (types.attrsOf unitOption) checkService; + description = lib.mdDoc '' + Each attribute in this set specifies an option in the + `[Service]` section of the unit. See + {manpage}`systemd.service(5)` for details. + ''; + }; + + script = mkOption { + type = types.lines; + default = ""; + description = lib.mdDoc "Shell commands executed as the service's main process."; + }; + + scriptArgs = mkOption { + type = types.str; + default = ""; + example = "%i"; + description = lib.mdDoc '' + Arguments passed to the main process script. + Can contain specifiers (`%` placeholders expanded by systemd, see {manpage}`systemd.unit(5)`). + ''; + }; + + preStart = mkOption { + type = types.lines; + default = ""; + description = lib.mdDoc '' + Shell commands executed before the service's main process + is started. + ''; + }; + + postStart = mkOption { + type = types.lines; + default = ""; + description = lib.mdDoc '' + Shell commands executed after the service's main process + is started. + ''; + }; + + reload = mkOption { + type = types.lines; + default = ""; + description = lib.mdDoc '' + Shell commands executed when the service's main process + is reloaded. + ''; + }; + + preStop = mkOption { + type = types.lines; + default = ""; + description = lib.mdDoc '' + Shell commands executed to stop the service. + ''; + }; + + postStop = mkOption { + type = types.lines; + default = ""; + description = lib.mdDoc '' + Shell commands executed after the service's main process + has exited. + ''; + }; + + jobScripts = mkOption { + type = with types; coercedTo path singleton (listOf path); + internal = true; + description = lib.mdDoc "A list of all job script derivations of this unit."; + default = []; + }; + + }; + + config = mkMerge [ + (mkIf (config.preStart != "") rec { + jobScripts = makeJobScript "${name}-pre-start" config.preStart; + serviceConfig.ExecStartPre = [ jobScripts ]; + }) + (mkIf (config.script != "") rec { + jobScripts = makeJobScript "${name}-start" config.script; + serviceConfig.ExecStart = jobScripts + " " + config.scriptArgs; + }) + (mkIf (config.postStart != "") rec { + jobScripts = (makeJobScript "${name}-post-start" config.postStart); + serviceConfig.ExecStartPost = [ jobScripts ]; + }) + (mkIf (config.reload != "") rec { + jobScripts = makeJobScript "${name}-reload" config.reload; + serviceConfig.ExecReload = jobScripts; + }) + (mkIf (config.preStop != "") rec { + jobScripts = makeJobScript "${name}-pre-stop" config.preStop; + serviceConfig.ExecStop = jobScripts; + }) + (mkIf (config.postStop != "") rec { + jobScripts = makeJobScript "${name}-post-stop" config.postStop; + serviceConfig.ExecStopPost = jobScripts; + }) + ]; + + }; + + stage2ServiceOptions = { + imports = [ + stage2CommonUnitOptions + serviceOptions + ]; + + options = { + restartIfChanged = mkOption { + type = types.bool; + default = true; + description = lib.mdDoc '' + Whether the service should be restarted during a NixOS + configuration switch if its definition has changed. + ''; + }; + + reloadIfChanged = mkOption { + type = types.bool; + default = false; + description = lib.mdDoc '' + Whether the service should be reloaded during a NixOS + configuration switch if its definition has changed. If + enabled, the value of {option}`restartIfChanged` is + ignored. + + This option should not be used anymore in favor of + {option}`reloadTriggers` which allows more granular + control of when a service is reloaded and when a service + is restarted. + ''; + }; + + stopIfChanged = mkOption { + type = types.bool; + default = true; + description = lib.mdDoc '' + If set, a changed unit is restarted by calling + {command}`systemctl stop` in the old configuration, + then {command}`systemctl start` in the new one. + Otherwise, it is restarted in a single step using + {command}`systemctl restart` in the new configuration. + The latter is less correct because it runs the + `ExecStop` commands from the new + configuration. + ''; + }; + + startAt = mkOption { + type = with types; either str (listOf str); + default = []; + example = "Sun 14:00:00"; + description = lib.mdDoc '' + Automatically start this unit at the given date/time, which + must be in the format described in + {manpage}`systemd.time(7)`. This is equivalent + to adding a corresponding timer unit with + {option}`OnCalendar` set to the value given here. + ''; + apply = v: if isList v then v else [ v ]; + }; + }; + }; + + stage1ServiceOptions = { + imports = [ + stage1CommonUnitOptions + serviceOptions + ]; + }; + + + socketOptions = { + options = { + + listenStreams = mkOption { + default = []; + type = types.listOf types.str; + example = [ "0.0.0.0:993" "/run/my-socket" ]; + description = lib.mdDoc '' + For each item in this list, a `ListenStream` + option in the `[Socket]` section will be created. + ''; + }; + + listenDatagrams = mkOption { + default = []; + type = types.listOf types.str; + example = [ "0.0.0.0:993" "/run/my-socket" ]; + description = lib.mdDoc '' + For each item in this list, a `ListenDatagram` + option in the `[Socket]` section will be created. + ''; + }; + + socketConfig = mkOption { + default = {}; + example = { ListenStream = "/run/my-socket"; }; + type = types.attrsOf unitOption; + description = lib.mdDoc '' + Each attribute in this set specifies an option in the + `[Socket]` section of the unit. See + {manpage}`systemd.socket(5)` for details. + ''; + }; + }; + + }; + + stage2SocketOptions = { + imports = [ + stage2CommonUnitOptions + socketOptions + ]; + }; + + stage1SocketOptions = { + imports = [ + stage1CommonUnitOptions + socketOptions + ]; + }; + + + timerOptions = { + options = { + + timerConfig = mkOption { + default = {}; + example = { OnCalendar = "Sun 14:00:00"; Unit = "foo.service"; }; + type = types.attrsOf unitOption; + description = lib.mdDoc '' + Each attribute in this set specifies an option in the + `[Timer]` section of the unit. See + {manpage}`systemd.timer(5)` and + {manpage}`systemd.time(7)` for details. + ''; + }; + + }; + }; + + stage2TimerOptions = { + imports = [ + stage2CommonUnitOptions + timerOptions + ]; + }; + + stage1TimerOptions = { + imports = [ + stage1CommonUnitOptions + timerOptions + ]; + }; + + + pathOptions = { + options = { + + pathConfig = mkOption { + default = {}; + example = { PathChanged = "/some/path"; Unit = "changedpath.service"; }; + type = types.attrsOf unitOption; + description = lib.mdDoc '' + Each attribute in this set specifies an option in the + `[Path]` section of the unit. See + {manpage}`systemd.path(5)` for details. + ''; + }; + + }; + }; + + stage2PathOptions = { + imports = [ + stage2CommonUnitOptions + pathOptions + ]; + }; + + stage1PathOptions = { + imports = [ + stage1CommonUnitOptions + pathOptions + ]; + }; + + + mountOptions = { + options = { + + what = mkOption { + example = "/dev/sda1"; + type = types.str; + description = lib.mdDoc "Absolute path of device node, file or other resource. (Mandatory)"; + }; + + where = mkOption { + example = "/mnt"; + type = types.str; + description = lib.mdDoc '' + Absolute path of a directory of the mount point. + Will be created if it doesn't exist. (Mandatory) + ''; + }; + + type = mkOption { + default = ""; + example = "ext4"; + type = types.str; + description = lib.mdDoc "File system type."; + }; + + options = mkOption { + default = ""; + example = "noatime"; + type = types.commas; + description = lib.mdDoc "Options used to mount the file system."; + }; + + mountConfig = mkOption { + default = {}; + example = { DirectoryMode = "0775"; }; + type = types.attrsOf unitOption; + description = lib.mdDoc '' + Each attribute in this set specifies an option in the + `[Mount]` section of the unit. See + {manpage}`systemd.mount(5)` for details. + ''; + }; + + }; + }; + + stage2MountOptions = { + imports = [ + stage2CommonUnitOptions + mountOptions + ]; + }; + + stage1MountOptions = { + imports = [ + stage1CommonUnitOptions + mountOptions + ]; + }; + + automountOptions = { + options = { + + where = mkOption { + example = "/mnt"; + type = types.str; + description = lib.mdDoc '' + Absolute path of a directory of the mount point. + Will be created if it doesn't exist. (Mandatory) + ''; + }; + + automountConfig = mkOption { + default = {}; + example = { DirectoryMode = "0775"; }; + type = types.attrsOf unitOption; + description = lib.mdDoc '' + Each attribute in this set specifies an option in the + `[Automount]` section of the unit. See + {manpage}`systemd.automount(5)` for details. + ''; + }; + + }; + }; + + stage2AutomountOptions = { + imports = [ + stage2CommonUnitOptions + automountOptions + ]; + }; + + stage1AutomountOptions = { + imports = [ + stage1CommonUnitOptions + automountOptions + ]; + }; + + sliceOptions = { + options = { + + sliceConfig = mkOption { + default = {}; + example = { MemoryMax = "2G"; }; + type = types.attrsOf unitOption; + description = lib.mdDoc '' + Each attribute in this set specifies an option in the + `[Slice]` section of the unit. See + {manpage}`systemd.slice(5)` for details. + ''; + }; + + }; + }; + + stage2SliceOptions = { + imports = [ + stage2CommonUnitOptions + sliceOptions + ]; + }; + + stage1SliceOptions = { + imports = [ + stage1CommonUnitOptions + sliceOptions + ]; + }; + +} diff --git a/nixpkgs/nixos/lib/test-driver/default.nix b/nixpkgs/nixos/lib/test-driver/default.nix new file mode 100644 index 000000000000..1acdaacc4e65 --- /dev/null +++ b/nixpkgs/nixos/lib/test-driver/default.nix @@ -0,0 +1,53 @@ +{ lib +, python3Packages +, enableOCR ? false +, qemu_pkg ? qemu_test +, coreutils +, imagemagick_light +, netpbm +, qemu_test +, socat +, ruff +, tesseract4 +, vde2 +, extraPythonPackages ? (_ : []) +, nixosTests +}: + +python3Packages.buildPythonApplication { + pname = "nixos-test-driver"; + version = "1.1"; + src = ./.; + pyproject = true; + + propagatedBuildInputs = [ + coreutils + netpbm + python3Packages.colorama + python3Packages.ptpython + qemu_pkg + socat + vde2 + ] + ++ (lib.optionals enableOCR [ imagemagick_light tesseract4 ]) + ++ extraPythonPackages python3Packages; + + nativeBuildInputs = [ + python3Packages.setuptools + ]; + + passthru.tests = { + inherit (nixosTests.nixos-test-driver) driver-timeout; + }; + + doCheck = true; + nativeCheckInputs = with python3Packages; [ mypy ruff black ]; + checkPhase = '' + echo -e "\x1b[32m## run mypy\x1b[0m" + mypy test_driver extract-docstrings.py + echo -e "\x1b[32m## run ruff\x1b[0m" + ruff . + echo -e "\x1b[32m## run black\x1b[0m" + black --check --diff . + ''; +} diff --git a/nixpkgs/nixos/lib/test-driver/extract-docstrings.py b/nixpkgs/nixos/lib/test-driver/extract-docstrings.py new file mode 100644 index 000000000000..64850ca711f3 --- /dev/null +++ b/nixpkgs/nixos/lib/test-driver/extract-docstrings.py @@ -0,0 +1,74 @@ +import ast +import sys +from pathlib import Path + +""" +This program takes all the Machine class methods and prints its methods in +markdown-style. These can then be included in the NixOS test driver +markdown style, assuming the docstrings themselves are also in markdown. + +These are included in the test driver documentation in the NixOS manual. +See https://nixos.org/manual/nixos/stable/#ssec-machine-objects + +The python input looks like this: + +```py +... + +class Machine(...): + ... + + def some_function(self, param1, param2): + "" + documentation string of some_function. + foo bar baz. + "" + ... +``` + +Output will be: + +```markdown +... + +some_function(param1, param2) + +: documentation string of some_function. + foo bar baz. + +... +``` + +""" + + +def main() -> None: + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} <path-to-test-driver>") + sys.exit(1) + + module = ast.parse(Path(sys.argv[1]).read_text()) + + class_definitions = (node for node in module.body if isinstance(node, ast.ClassDef)) + + machine_class = next(filter(lambda x: x.name == "Machine", class_definitions)) + assert machine_class is not None + + function_definitions = [ + node for node in machine_class.body if isinstance(node, ast.FunctionDef) + ] + function_definitions.sort(key=lambda x: x.name) + + for function in function_definitions: + docstr = ast.get_docstring(function) + if docstr is not None: + args = ", ".join(a.arg for a in function.args.args[1:]) + args = f"({args})" + + docstr = "\n".join(f" {line}" for line in docstr.strip().splitlines()) + + print(f"{function.name}{args}\n\n:{docstr[1:]}\n") + + +if __name__ == "__main__": + main() diff --git a/nixpkgs/nixos/lib/test-driver/nixos-test-driver-docstrings.nix b/nixpkgs/nixos/lib/test-driver/nixos-test-driver-docstrings.nix new file mode 100644 index 000000000000..a3ef50e4e820 --- /dev/null +++ b/nixpkgs/nixos/lib/test-driver/nixos-test-driver-docstrings.nix @@ -0,0 +1,13 @@ +{ runCommand +, python3 +}: + +let + env = { nativeBuildInputs = [ python3 ]; }; +in + +runCommand "nixos-test-driver-docstrings" env '' + mkdir $out + python3 ${./extract-docstrings.py} ${./test_driver/machine.py} \ + > $out/machine-methods.md +'' diff --git a/nixpkgs/nixos/lib/test-driver/pyproject.toml b/nixpkgs/nixos/lib/test-driver/pyproject.toml new file mode 100644 index 000000000000..8638f14dfdae --- /dev/null +++ b/nixpkgs/nixos/lib/test-driver/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "nixos-test-driver" +version = "0.0.0" + +[project.scripts] +nixos-test-driver = "test_driver:main" +generate-driver-symbols = "test_driver:generate_driver_symbols" + +[tool.setuptools.packages] +find = {} + +[tool.setuptools.package-data] +test_driver = ["py.typed"] + +[tool.ruff] +line-length = 88 + +select = ["E", "F", "I", "U", "N"] +ignore = ["E501"] + +# xxx: we can import https://pypi.org/project/types-colorama/ here +[[tool.mypy.overrides]] +module = "colorama.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "ptpython.*" +ignore_missing_imports = true + +[tool.black] +line-length = 88 +target-version = ['py39'] +include = '\.pyi?$' + +[tool.mypy] +python_version = "3.10" +warn_redundant_casts = true +disallow_untyped_calls = true +disallow_untyped_defs = true +no_implicit_optional = true diff --git a/nixpkgs/nixos/lib/test-driver/shell.nix b/nixpkgs/nixos/lib/test-driver/shell.nix new file mode 100644 index 000000000000..367bbad556c0 --- /dev/null +++ b/nixpkgs/nixos/lib/test-driver/shell.nix @@ -0,0 +1,2 @@ +with import ../../.. {}; +pkgs.callPackage ./default.nix {} diff --git a/nixpkgs/nixos/lib/test-driver/test_driver/__init__.py b/nixpkgs/nixos/lib/test-driver/test_driver/__init__.py new file mode 100755 index 000000000000..9daae1e941a6 --- /dev/null +++ b/nixpkgs/nixos/lib/test-driver/test_driver/__init__.py @@ -0,0 +1,140 @@ +import argparse +import os +import time +from pathlib import Path + +import ptpython.repl + +from test_driver.driver import Driver +from test_driver.logger import rootlog + + +class EnvDefault(argparse.Action): + """An argpars Action that takes values from the specified + environment variable as the flags default value. + """ + + def __init__(self, envvar, required=False, default=None, nargs=None, **kwargs): # type: ignore + if not default and envvar: + if envvar in os.environ: + if nargs is not None and (nargs.isdigit() or nargs in ["*", "+"]): + default = os.environ[envvar].split() + else: + default = os.environ[envvar] + kwargs["help"] = ( + kwargs["help"] + f" (default from environment: {default})" + ) + if required and default: + required = False + super().__init__(default=default, required=required, nargs=nargs, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): # type: ignore + setattr(namespace, self.dest, values) + + +def writeable_dir(arg: str) -> Path: + """Raises an ArgumentTypeError if the given argument isn't a writeable directory + Note: We want to fail as early as possible if a directory isn't writeable, + since an executed nixos-test could fail (very late) because of the test-driver + writing in a directory without proper permissions. + """ + path = Path(arg) + if not path.is_dir(): + raise argparse.ArgumentTypeError(f"{path} is not a directory") + if not os.access(path, os.W_OK): + raise argparse.ArgumentTypeError(f"{path} is not a writeable directory") + return path + + +def main() -> None: + arg_parser = argparse.ArgumentParser(prog="nixos-test-driver") + arg_parser.add_argument( + "-K", + "--keep-vm-state", + help="re-use a VM state coming from a previous run", + action="store_true", + ) + arg_parser.add_argument( + "-I", + "--interactive", + help="drop into a python repl and run the tests interactively", + action=argparse.BooleanOptionalAction, + ) + arg_parser.add_argument( + "--start-scripts", + metavar="START-SCRIPT", + action=EnvDefault, + envvar="startScripts", + nargs="*", + help="start scripts for participating virtual machines", + ) + arg_parser.add_argument( + "--vlans", + metavar="VLAN", + action=EnvDefault, + envvar="vlans", + nargs="*", + help="vlans to span by the driver", + ) + arg_parser.add_argument( + "--global-timeout", + type=int, + metavar="GLOBAL_TIMEOUT", + action=EnvDefault, + envvar="globalTimeout", + help="Timeout in seconds for the whole test", + ) + arg_parser.add_argument( + "-o", + "--output_directory", + help="""The path to the directory where outputs copied from the VM will be placed. + By e.g. Machine.copy_from_vm or Machine.screenshot""", + default=Path.cwd(), + type=writeable_dir, + ) + arg_parser.add_argument( + "testscript", + action=EnvDefault, + envvar="testScript", + help="the test script to run", + type=Path, + ) + + args = arg_parser.parse_args() + + if not args.keep_vm_state: + rootlog.info("Machine state will be reset. To keep it, pass --keep-vm-state") + + with Driver( + args.start_scripts, + args.vlans, + args.testscript.read_text(), + args.output_directory.resolve(), + args.keep_vm_state, + args.global_timeout, + ) as driver: + if args.interactive: + history_dir = os.getcwd() + history_path = os.path.join(history_dir, ".nixos-test-history") + ptpython.repl.embed( + driver.test_symbols(), + {}, + history_filename=history_path, + ) + else: + tic = time.time() + driver.run_tests() + toc = time.time() + rootlog.info(f"test script finished in {(toc-tic):.2f}s") + + +def generate_driver_symbols() -> None: + """ + This generates a file with symbols of the test-driver code that can be used + in user's test scripts. That list is then used by pyflakes to lint those + scripts. + """ + d = Driver([], [], "", Path()) + test_symbols = d.test_symbols() + with open("driver-symbols", "w") as fp: + fp.write(",".join(test_symbols.keys())) diff --git a/nixpkgs/nixos/lib/test-driver/test_driver/driver.py b/nixpkgs/nixos/lib/test-driver/test_driver/driver.py new file mode 100644 index 000000000000..786821b0cc0d --- /dev/null +++ b/nixpkgs/nixos/lib/test-driver/test_driver/driver.py @@ -0,0 +1,260 @@ +import os +import re +import signal +import tempfile +import threading +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Callable, ContextManager, Dict, Iterator, List, Optional, Union + +from test_driver.logger import rootlog +from test_driver.machine import Machine, NixStartScript, retry +from test_driver.polling_condition import PollingCondition +from test_driver.vlan import VLan + + +def get_tmp_dir() -> Path: + """Returns a temporary directory that is defined by TMPDIR, TEMP, TMP or CWD + Raises an exception in case the retrieved temporary directory is not writeable + See https://docs.python.org/3/library/tempfile.html#tempfile.gettempdir + """ + tmp_dir = Path(tempfile.gettempdir()) + tmp_dir.mkdir(mode=0o700, exist_ok=True) + if not tmp_dir.is_dir(): + raise NotADirectoryError( + f"The directory defined by TMPDIR, TEMP, TMP or CWD: {tmp_dir} is not a directory" + ) + if not os.access(tmp_dir, os.W_OK): + raise PermissionError( + f"The directory defined by TMPDIR, TEMP, TMP, or CWD: {tmp_dir} is not writeable" + ) + return tmp_dir + + +def pythonize_name(name: str) -> str: + return re.sub(r"^[^A-z_]|[^A-z0-9_]", "_", name) + + +class Driver: + """A handle to the driver that sets up the environment + and runs the tests""" + + tests: str + vlans: List[VLan] + machines: List[Machine] + polling_conditions: List[PollingCondition] + global_timeout: int + race_timer: threading.Timer + + def __init__( + self, + start_scripts: List[str], + vlans: List[int], + tests: str, + out_dir: Path, + keep_vm_state: bool = False, + global_timeout: int = 24 * 60 * 60 * 7, + ): + self.tests = tests + self.out_dir = out_dir + self.global_timeout = global_timeout + self.race_timer = threading.Timer(global_timeout, self.terminate_test) + + tmp_dir = get_tmp_dir() + + with rootlog.nested("start all VLans"): + vlans = list(set(vlans)) + self.vlans = [VLan(nr, tmp_dir) for nr in vlans] + + def cmd(scripts: List[str]) -> Iterator[NixStartScript]: + for s in scripts: + yield NixStartScript(s) + + self.polling_conditions = [] + + self.machines = [ + Machine( + start_command=cmd, + keep_vm_state=keep_vm_state, + name=cmd.machine_name, + tmp_dir=tmp_dir, + callbacks=[self.check_polling_conditions], + out_dir=self.out_dir, + ) + for cmd in cmd(start_scripts) + ] + + def __enter__(self) -> "Driver": + return self + + def __exit__(self, *_: Any) -> None: + with rootlog.nested("cleanup"): + self.race_timer.cancel() + for machine in self.machines: + machine.release() + + def subtest(self, name: str) -> Iterator[None]: + """Group logs under a given test name""" + with rootlog.nested("subtest: " + name): + try: + yield + return True + except Exception as e: + rootlog.error(f'Test "{name}" failed with error: "{e}"') + raise e + + def test_symbols(self) -> Dict[str, Any]: + @contextmanager + def subtest(name: str) -> Iterator[None]: + return self.subtest(name) + + general_symbols = dict( + start_all=self.start_all, + test_script=self.test_script, + machines=self.machines, + vlans=self.vlans, + driver=self, + log=rootlog, + os=os, + create_machine=self.create_machine, + subtest=subtest, + run_tests=self.run_tests, + join_all=self.join_all, + retry=retry, + serial_stdout_off=self.serial_stdout_off, + serial_stdout_on=self.serial_stdout_on, + polling_condition=self.polling_condition, + Machine=Machine, # for typing + ) + machine_symbols = {pythonize_name(m.name): m for m in self.machines} + # If there's exactly one machine, make it available under the name + # "machine", even if it's not called that. + if len(self.machines) == 1: + (machine_symbols["machine"],) = self.machines + vlan_symbols = { + f"vlan{v.nr}": self.vlans[idx] for idx, v in enumerate(self.vlans) + } + print( + "additionally exposed symbols:\n " + + ", ".join(map(lambda m: m.name, self.machines)) + + ",\n " + + ", ".join(map(lambda v: f"vlan{v.nr}", self.vlans)) + + ",\n " + + ", ".join(list(general_symbols.keys())) + ) + return {**general_symbols, **machine_symbols, **vlan_symbols} + + def test_script(self) -> None: + """Run the test script""" + with rootlog.nested("run the VM test script"): + symbols = self.test_symbols() # call eagerly + exec(self.tests, symbols, None) + + def run_tests(self) -> None: + """Run the test script (for non-interactive test runs)""" + rootlog.info( + f"Test will time out and terminate in {self.global_timeout} seconds" + ) + self.race_timer.start() + self.test_script() + # TODO: Collect coverage data + for machine in self.machines: + if machine.is_up(): + machine.execute("sync") + + def start_all(self) -> None: + """Start all machines""" + with rootlog.nested("start all VMs"): + for machine in self.machines: + machine.start() + + def join_all(self) -> None: + """Wait for all machines to shut down""" + with rootlog.nested("wait for all VMs to finish"): + for machine in self.machines: + machine.wait_for_shutdown() + self.race_timer.cancel() + + def terminate_test(self) -> None: + # This will be usually running in another thread than + # the thread actually executing the test script. + with rootlog.nested("timeout reached; test terminating..."): + for machine in self.machines: + machine.release() + # As we cannot `sys.exit` from another thread + # We can at least force the main thread to get SIGTERM'ed. + # This will prevent any user who caught all the exceptions + # to swallow them and prevent itself from terminating. + os.kill(os.getpid(), signal.SIGTERM) + + def create_machine(self, args: Dict[str, Any]) -> Machine: + tmp_dir = get_tmp_dir() + + if args.get("startCommand"): + start_command: str = args.get("startCommand", "") + cmd = NixStartScript(start_command) + name = args.get("name", cmd.machine_name) + else: + cmd = Machine.create_startcommand(args) # type: ignore + name = args.get("name", "machine") + + return Machine( + tmp_dir=tmp_dir, + out_dir=self.out_dir, + start_command=cmd, + name=name, + keep_vm_state=args.get("keep_vm_state", False), + ) + + def serial_stdout_on(self) -> None: + rootlog._print_serial_logs = True + + def serial_stdout_off(self) -> None: + rootlog._print_serial_logs = False + + def check_polling_conditions(self) -> None: + for condition in self.polling_conditions: + condition.maybe_raise() + + def polling_condition( + self, + fun_: Optional[Callable] = None, + *, + seconds_interval: float = 2.0, + description: Optional[str] = None, + ) -> Union[Callable[[Callable], ContextManager], ContextManager]: + driver = self + + class Poll: + def __init__(self, fun: Callable): + self.condition = PollingCondition( + fun, + seconds_interval, + description, + ) + + def __enter__(self) -> None: + driver.polling_conditions.append(self.condition) + + def __exit__(self, a, b, c) -> None: # type: ignore + res = driver.polling_conditions.pop() + assert res is self.condition + + def wait(self, timeout: int = 900) -> None: + def condition(last: bool) -> bool: + if last: + rootlog.info(f"Last chance for {self.condition.description}") + ret = self.condition.check(force=True) + if not ret and not last: + rootlog.info( + f"({self.condition.description} failure not fatal yet)" + ) + return ret + + with rootlog.nested(f"waiting for {self.condition.description}"): + retry(condition, timeout=timeout) + + if fun_ is None: + return Poll + else: + return Poll(fun_) diff --git a/nixpkgs/nixos/lib/test-driver/test_driver/logger.py b/nixpkgs/nixos/lib/test-driver/test_driver/logger.py new file mode 100644 index 000000000000..116244b5e4ae --- /dev/null +++ b/nixpkgs/nixos/lib/test-driver/test_driver/logger.py @@ -0,0 +1,107 @@ +# mypy: disable-error-code="no-untyped-call" +# drop the above line when mypy is upgraded to include +# https://github.com/python/typeshed/commit/49b717ca52bf0781a538b04c0d76a5513f7119b8 +import codecs +import os +import sys +import time +import unicodedata +from contextlib import contextmanager +from queue import Empty, Queue +from typing import Any, Dict, Iterator +from xml.sax.saxutils import XMLGenerator + +from colorama import Fore, Style + + +class Logger: + def __init__(self) -> None: + self.logfile = os.environ.get("LOGFILE", "/dev/null") + self.logfile_handle = codecs.open(self.logfile, "wb") + self.xml = XMLGenerator(self.logfile_handle, encoding="utf-8") + self.queue: "Queue[Dict[str, str]]" = Queue() + + self.xml.startDocument() + self.xml.startElement("logfile", attrs={}) + + self._print_serial_logs = True + + @staticmethod + def _eprint(*args: object, **kwargs: Any) -> None: + print(*args, file=sys.stderr, **kwargs) + + def close(self) -> None: + self.xml.endElement("logfile") + self.xml.endDocument() + self.logfile_handle.close() + + def sanitise(self, message: str) -> str: + return "".join(ch for ch in message if unicodedata.category(ch)[0] != "C") + + def maybe_prefix(self, message: str, attributes: Dict[str, str]) -> str: + if "machine" in attributes: + return f"{attributes['machine']}: {message}" + return message + + def log_line(self, message: str, attributes: Dict[str, str]) -> None: + self.xml.startElement("line", attributes) + self.xml.characters(message) + self.xml.endElement("line") + + def info(self, *args, **kwargs) -> None: # type: ignore + self.log(*args, **kwargs) + + def warning(self, *args, **kwargs) -> None: # type: ignore + self.log(*args, **kwargs) + + def error(self, *args, **kwargs) -> None: # type: ignore + self.log(*args, **kwargs) + sys.exit(1) + + def log(self, message: str, attributes: Dict[str, str] = {}) -> None: + self._eprint(self.maybe_prefix(message, attributes)) + self.drain_log_queue() + self.log_line(message, attributes) + + def log_serial(self, message: str, machine: str) -> None: + self.enqueue({"msg": message, "machine": machine, "type": "serial"}) + if self._print_serial_logs: + self._eprint(Style.DIM + f"{machine} # {message}" + Style.RESET_ALL) + + def enqueue(self, item: Dict[str, str]) -> None: + self.queue.put(item) + + def drain_log_queue(self) -> None: + try: + while True: + item = self.queue.get_nowait() + msg = self.sanitise(item["msg"]) + del item["msg"] + self.log_line(msg, item) + except Empty: + pass + + @contextmanager + def nested(self, message: str, attributes: Dict[str, str] = {}) -> Iterator[None]: + self._eprint( + self.maybe_prefix( + Style.BRIGHT + Fore.GREEN + message + Style.RESET_ALL, attributes + ) + ) + + self.xml.startElement("nest", attrs={}) + self.xml.startElement("head", attributes) + self.xml.characters(message) + self.xml.endElement("head") + + tic = time.time() + self.drain_log_queue() + yield + self.drain_log_queue() + toc = time.time() + self.log(f"(finished: {message}, in {toc - tic:.2f} seconds)") + + self.xml.endElement("nest") + + +rootlog = Logger() diff --git a/nixpkgs/nixos/lib/test-driver/test_driver/machine.py b/nixpkgs/nixos/lib/test-driver/test_driver/machine.py new file mode 100644 index 000000000000..da60b669fa27 --- /dev/null +++ b/nixpkgs/nixos/lib/test-driver/test_driver/machine.py @@ -0,0 +1,1324 @@ +import base64 +import io +import os +import queue +import re +import select +import shlex +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +import time +from contextlib import _GeneratorContextManager, nullcontext +from pathlib import Path +from queue import Queue +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple + +from test_driver.logger import rootlog + +from .qmp import QMPSession + +CHAR_TO_KEY = { + "A": "shift-a", + "N": "shift-n", + "-": "0x0C", + "_": "shift-0x0C", + "B": "shift-b", + "O": "shift-o", + "=": "0x0D", + "+": "shift-0x0D", + "C": "shift-c", + "P": "shift-p", + "[": "0x1A", + "{": "shift-0x1A", + "D": "shift-d", + "Q": "shift-q", + "]": "0x1B", + "}": "shift-0x1B", + "E": "shift-e", + "R": "shift-r", + ";": "0x27", + ":": "shift-0x27", + "F": "shift-f", + "S": "shift-s", + "'": "0x28", + '"': "shift-0x28", + "G": "shift-g", + "T": "shift-t", + "`": "0x29", + "~": "shift-0x29", + "H": "shift-h", + "U": "shift-u", + "\\": "0x2B", + "|": "shift-0x2B", + "I": "shift-i", + "V": "shift-v", + ",": "0x33", + "<": "shift-0x33", + "J": "shift-j", + "W": "shift-w", + ".": "0x34", + ">": "shift-0x34", + "K": "shift-k", + "X": "shift-x", + "/": "0x35", + "?": "shift-0x35", + "L": "shift-l", + "Y": "shift-y", + " ": "spc", + "M": "shift-m", + "Z": "shift-z", + "\n": "ret", + "!": "shift-0x02", + "@": "shift-0x03", + "#": "shift-0x04", + "$": "shift-0x05", + "%": "shift-0x06", + "^": "shift-0x07", + "&": "shift-0x08", + "*": "shift-0x09", + "(": "shift-0x0A", + ")": "shift-0x0B", +} + + +def make_command(args: list) -> str: + return " ".join(map(shlex.quote, (map(str, args)))) + + +def _perform_ocr_on_screenshot( + screenshot_path: str, model_ids: Iterable[int] +) -> List[str]: + if shutil.which("tesseract") is None: + raise Exception("OCR requested but enableOCR is false") + + magick_args = ( + "-filter Catrom -density 72 -resample 300 " + + "-contrast -normalize -despeckle -type grayscale " + + "-sharpen 1 -posterize 3 -negate -gamma 100 " + + "-blur 1x65535" + ) + + tess_args = "-c debug_file=/dev/null --psm 11" + + cmd = f"convert {magick_args} '{screenshot_path}' 'tiff:{screenshot_path}.tiff'" + ret = subprocess.run(cmd, shell=True, capture_output=True) + if ret.returncode != 0: + raise Exception(f"TIFF conversion failed with exit code {ret.returncode}") + + model_results = [] + for model_id in model_ids: + cmd = f"tesseract '{screenshot_path}.tiff' - {tess_args} --oem '{model_id}'" + ret = subprocess.run(cmd, shell=True, capture_output=True) + if ret.returncode != 0: + raise Exception(f"OCR failed with exit code {ret.returncode}") + model_results.append(ret.stdout.decode("utf-8")) + + return model_results + + +def retry(fn: Callable, timeout: int = 900) -> None: + """Call the given function repeatedly, with 1 second intervals, + until it returns True or a timeout is reached. + """ + + for _ in range(timeout): + if fn(False): + return + time.sleep(1) + + if not fn(True): + raise Exception(f"action timed out after {timeout} seconds") + + +class StartCommand: + """The Base Start Command knows how to append the necessary + runtime qemu options as determined by a particular test driver + run. Any such start command is expected to happily receive and + append additional qemu args. + """ + + _cmd: str + + def cmd( + self, + monitor_socket_path: Path, + qmp_socket_path: Path, + shell_socket_path: Path, + allow_reboot: bool = False, + ) -> str: + display_opts = "" + display_available = any(x in os.environ for x in ["DISPLAY", "WAYLAND_DISPLAY"]) + if not display_available: + display_opts += " -nographic" + + # qemu options + qemu_opts = ( + " -device virtio-serial" + # Note: virtconsole will map to /dev/hvc0 in Linux guests + " -device virtconsole,chardev=shell" + " -device virtio-rng-pci" + " -serial stdio" + ) + if not allow_reboot: + qemu_opts += " -no-reboot" + # TODO: qemu script already catpures this env variable, legacy? + qemu_opts += " " + os.environ.get("QEMU_OPTS", "") + + return ( + f"{self._cmd}" + f" -qmp unix:{qmp_socket_path},server=on,wait=off" + f" -monitor unix:{monitor_socket_path}" + f" -chardev socket,id=shell,path={shell_socket_path}" + f"{qemu_opts}" + f"{display_opts}" + ) + + @staticmethod + def build_environment( + state_dir: Path, + shared_dir: Path, + ) -> dict: + # We make a copy to not update the current environment + env = dict(os.environ) + env.update( + { + "TMPDIR": str(state_dir), + "SHARED_DIR": str(shared_dir), + "USE_TMPDIR": "1", + } + ) + return env + + def run( + self, + state_dir: Path, + shared_dir: Path, + monitor_socket_path: Path, + qmp_socket_path: Path, + shell_socket_path: Path, + allow_reboot: bool, + ) -> subprocess.Popen: + return subprocess.Popen( + self.cmd( + monitor_socket_path, qmp_socket_path, shell_socket_path, allow_reboot + ), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, + cwd=state_dir, + env=self.build_environment(state_dir, shared_dir), + ) + + +class NixStartScript(StartCommand): + """A start script from nixos/modules/virtualiation/qemu-vm.nix + that also satisfies the requirement of the BaseStartCommand. + These Nix commands have the particular characteristic that the + machine name can be extracted out of them via a regex match. + (Admittedly a _very_ implicit contract, evtl. TODO fix) + """ + + def __init__(self, script: str): + self._cmd = script + + @property + def machine_name(self) -> str: + match = re.search("run-(.+)-vm$", self._cmd) + name = "machine" + if match: + name = match.group(1) + return name + + +class LegacyStartCommand(StartCommand): + """Used in some places to create an ad-hoc machine instead of + using nix test instrumentation + module system for that purpose. + Legacy. + """ + + def __init__( + self, + netBackendArgs: Optional[str] = None, # noqa: N803 + netFrontendArgs: Optional[str] = None, # noqa: N803 + hda: Optional[Tuple[Path, str]] = None, + cdrom: Optional[str] = None, + usb: Optional[str] = None, + bios: Optional[str] = None, + qemuBinary: Optional[str] = None, # noqa: N803 + qemuFlags: Optional[str] = None, # noqa: N803 + ): + if qemuBinary is not None: + self._cmd = qemuBinary + else: + self._cmd = "qemu-kvm" + + self._cmd += " -m 384" + + # networking + net_backend = "-netdev user,id=net0" + net_frontend = "-device virtio-net-pci,netdev=net0" + if netBackendArgs is not None: + net_backend += "," + netBackendArgs + if netFrontendArgs is not None: + net_frontend += "," + netFrontendArgs + self._cmd += f" {net_backend} {net_frontend}" + + # hda + hda_cmd = "" + if hda is not None: + hda_path = hda[0].resolve() + hda_interface = hda[1] + if hda_interface == "scsi": + hda_cmd += ( + f" -drive id=hda,file={hda_path},werror=report,if=none" + " -device scsi-hd,drive=hda" + ) + else: + hda_cmd += f" -drive file={hda_path},if={hda_interface},werror=report" + self._cmd += hda_cmd + + # cdrom + if cdrom is not None: + self._cmd += f" -cdrom {cdrom}" + + # usb + usb_cmd = "" + if usb is not None: + # https://github.com/qemu/qemu/blob/master/docs/usb2.txt + usb_cmd += ( + " -device usb-ehci" + f" -drive id=usbdisk,file={usb},if=none,readonly" + " -device usb-storage,drive=usbdisk " + ) + self._cmd += usb_cmd + + # bios + if bios is not None: + self._cmd += f" -bios {bios}" + + # qemu flags + if qemuFlags is not None: + self._cmd += f" {qemuFlags}" + + +class Machine: + """A handle to the machine with this name, that also knows how to manage + the machine lifecycle with the help of a start script / command.""" + + name: str + out_dir: Path + tmp_dir: Path + shared_dir: Path + state_dir: Path + monitor_path: Path + qmp_path: Path + shell_path: Path + + start_command: StartCommand + keep_vm_state: bool + + process: Optional[subprocess.Popen] + pid: Optional[int] + monitor: Optional[socket.socket] + qmp_client: Optional[QMPSession] + shell: Optional[socket.socket] + serial_thread: Optional[threading.Thread] + + booted: bool + connected: bool + # Store last serial console lines for use + # of wait_for_console_text + last_lines: Queue = Queue() + callbacks: List[Callable] + + def __repr__(self) -> str: + return f"<Machine '{self.name}'>" + + def __init__( + self, + out_dir: Path, + tmp_dir: Path, + start_command: StartCommand, + name: str = "machine", + keep_vm_state: bool = False, + callbacks: Optional[List[Callable]] = None, + ) -> None: + self.out_dir = out_dir + self.tmp_dir = tmp_dir + self.keep_vm_state = keep_vm_state + self.name = name + self.start_command = start_command + self.callbacks = callbacks if callbacks is not None else [] + + # set up directories + self.shared_dir = self.tmp_dir / "shared-xchg" + self.shared_dir.mkdir(mode=0o700, exist_ok=True) + + self.state_dir = self.tmp_dir / f"vm-state-{self.name}" + self.monitor_path = self.state_dir / "monitor" + self.qmp_path = self.state_dir / "qmp" + self.shell_path = self.state_dir / "shell" + if (not self.keep_vm_state) and self.state_dir.exists(): + self.cleanup_statedir() + self.state_dir.mkdir(mode=0o700, exist_ok=True) + + self.process = None + self.pid = None + self.monitor = None + self.qmp_client = None + self.shell = None + self.serial_thread = None + + self.booted = False + self.connected = False + + @staticmethod + def create_startcommand(args: Dict[str, str]) -> StartCommand: + rootlog.warning( + "Using legacy create_startcommand(), " + "please use proper nix test vm instrumentation, instead " + "to generate the appropriate nixos test vm qemu startup script" + ) + hda = None + if args.get("hda"): + hda_arg: str = args.get("hda", "") + hda_arg_path: Path = Path(hda_arg) + hda = (hda_arg_path, args.get("hdaInterface", "")) + return LegacyStartCommand( + netBackendArgs=args.get("netBackendArgs"), + netFrontendArgs=args.get("netFrontendArgs"), + hda=hda, + cdrom=args.get("cdrom"), + usb=args.get("usb"), + bios=args.get("bios"), + qemuBinary=args.get("qemuBinary"), + qemuFlags=args.get("qemuFlags"), + ) + + def is_up(self) -> bool: + return self.booted and self.connected + + def log(self, msg: str) -> None: + rootlog.log(msg, {"machine": self.name}) + + def log_serial(self, msg: str) -> None: + rootlog.log_serial(msg, self.name) + + def nested(self, msg: str, attrs: Dict[str, str] = {}) -> _GeneratorContextManager: + my_attrs = {"machine": self.name} + my_attrs.update(attrs) + return rootlog.nested(msg, my_attrs) + + def wait_for_monitor_prompt(self) -> str: + assert self.monitor is not None + answer = "" + while True: + undecoded_answer = self.monitor.recv(1024) + if not undecoded_answer: + break + answer += undecoded_answer.decode() + if answer.endswith("(qemu) "): + break + return answer + + def send_monitor_command(self, command: str) -> str: + """ + Send a command to the QEMU monitor. This allows attaching + virtual USB disks to a running machine, among other things. + """ + self.run_callbacks() + message = f"{command}\n".encode() + assert self.monitor is not None + self.monitor.send(message) + return self.wait_for_monitor_prompt() + + def wait_for_unit( + self, unit: str, user: Optional[str] = None, timeout: int = 900 + ) -> None: + """ + Wait for a systemd unit to get into "active" state. + Throws exceptions on "failed" and "inactive" states as well as after + timing out. + """ + + def check_active(_: Any) -> bool: + state = self.get_unit_property(unit, "ActiveState", user) + if state == "failed": + raise Exception(f'unit "{unit}" reached state "{state}"') + + if state == "inactive": + status, jobs = self.systemctl("list-jobs --full 2>&1", user) + if "No jobs" in jobs: + info = self.get_unit_info(unit, user) + if info["ActiveState"] == state: + raise Exception( + f'unit "{unit}" is inactive and there are no pending jobs' + ) + + return state == "active" + + with self.nested( + f"waiting for unit {unit}" + + (f" with user {user}" if user is not None else "") + ): + retry(check_active, timeout) + + def get_unit_info(self, unit: str, user: Optional[str] = None) -> Dict[str, str]: + status, lines = self.systemctl(f'--no-pager show "{unit}"', user) + if status != 0: + raise Exception( + f'retrieving systemctl info for unit "{unit}"' + + ("" if user is None else f' under user "{user}"') + + f" failed with exit code {status}" + ) + + line_pattern = re.compile(r"^([^=]+)=(.*)$") + + def tuple_from_line(line: str) -> Tuple[str, str]: + match = line_pattern.match(line) + assert match is not None + return match[1], match[2] + + return dict( + tuple_from_line(line) + for line in lines.split("\n") + if line_pattern.match(line) + ) + + def get_unit_property( + self, + unit: str, + property: str, + user: Optional[str] = None, + ) -> str: + status, lines = self.systemctl( + f'--no-pager show "{unit}" --property="{property}"', + user, + ) + if status != 0: + raise Exception( + f'retrieving systemctl property "{property}" for unit "{unit}"' + + ("" if user is None else f' under user "{user}"') + + f" failed with exit code {status}" + ) + + invalid_output_message = ( + f'systemctl show --property "{property}" "{unit}"' + f"produced invalid output: {lines}" + ) + + line_pattern = re.compile(r"^([^=]+)=(.*)$") + match = line_pattern.match(lines) + assert match is not None, invalid_output_message + + assert match[1] == property, invalid_output_message + return match[2] + + def systemctl(self, q: str, user: Optional[str] = None) -> Tuple[int, str]: + """ + Runs `systemctl` commands with optional support for + `systemctl --user` + + ```py + # run `systemctl list-jobs --no-pager` + machine.systemctl("list-jobs --no-pager") + + # spawn a shell for `any-user` and run + # `systemctl --user list-jobs --no-pager` + machine.systemctl("list-jobs --no-pager", "any-user") + ``` + """ + if user is not None: + q = q.replace("'", "\\'") + return self.execute( + f"su -l {user} --shell /bin/sh -c " + "$'XDG_RUNTIME_DIR=/run/user/`id -u` " + f"systemctl --user {q}'" + ) + return self.execute(f"systemctl {q}") + + def require_unit_state(self, unit: str, require_state: str = "active") -> None: + with self.nested( + f"checking if unit '{unit}' has reached state '{require_state}'" + ): + info = self.get_unit_info(unit) + state = info["ActiveState"] + if state != require_state: + raise Exception( + f"Expected unit '{unit}' to to be in state " + f"'{require_state}' but it is in state '{state}'" + ) + + def _next_newline_closed_block_from_shell(self) -> str: + assert self.shell + output_buffer = [] + while True: + # This receives up to 4096 bytes from the socket + chunk = self.shell.recv(4096) + if not chunk: + # Probably a broken pipe, return the output we have + break + + decoded = chunk.decode() + output_buffer += [decoded] + if decoded[-1] == "\n": + break + return "".join(output_buffer) + + def execute( + self, + command: str, + check_return: bool = True, + check_output: bool = True, + timeout: Optional[int] = 900, + ) -> Tuple[int, str]: + """ + Execute a shell command, returning a list `(status, stdout)`. + + Commands are run with `set -euo pipefail` set: + + - If several commands are separated by `;` and one fails, the + command as a whole will fail. + + - For pipelines, the last non-zero exit status will be returned + (if there is one; otherwise zero will be returned). + + - Dereferencing unset variables fails the command. + + - It will wait for stdout to be closed. + + If the command detaches, it must close stdout, as `execute` will wait + for this to consume all output reliably. This can be achieved by + redirecting stdout to stderr `>&2`, to `/dev/console`, `/dev/null` or + a file. Examples of detaching commands are `sleep 365d &`, where the + shell forks a new process that can write to stdout and `xclip -i`, where + the `xclip` command itself forks without closing stdout. + + Takes an optional parameter `check_return` that defaults to `True`. + Setting this parameter to `False` will not check for the return code + and return -1 instead. This can be used for commands that shut down + the VM and would therefore break the pipe that would be used for + retrieving the return code. + + A timeout for the command can be specified (in seconds) using the optional + `timeout` parameter, e.g., `execute(cmd, timeout=10)` or + `execute(cmd, timeout=None)`. The default is 900 seconds. + """ + self.run_callbacks() + self.connect() + + # Always run command with shell opts + command = f"set -euo pipefail; {command}" + + timeout_str = "" + if timeout is not None: + timeout_str = f"timeout {timeout}" + + # While sh is bash on NixOS, this is not the case for every distro. + # We explicitly call bash here to allow for the driver to boot other distros as well. + out_command = ( + f"{timeout_str} bash -c {shlex.quote(command)} | (base64 -w 0; echo)\n" + ) + + assert self.shell + self.shell.send(out_command.encode()) + + if not check_output: + return (-2, "") + + # Get the output + output = base64.b64decode(self._next_newline_closed_block_from_shell()) + + if not check_return: + return (-1, output.decode()) + + # Get the return code + self.shell.send(b"echo ${PIPESTATUS[0]}\n") + rc = int(self._next_newline_closed_block_from_shell().strip()) + + return (rc, output.decode(errors="replace")) + + def shell_interact(self, address: Optional[str] = None) -> None: + """ + Allows you to directly interact with the guest shell. This should + only be used during test development, not in production tests. + Killing the interactive session with `Ctrl-d` or `Ctrl-c` also ends + the guest session. + """ + self.connect() + + if address is None: + address = "READLINE,prompt=$ " + self.log("Terminal is ready (there is no initial prompt):") + + assert self.shell + try: + subprocess.run( + ["socat", address, f"FD:{self.shell.fileno()}"], + pass_fds=[self.shell.fileno()], + ) + # allow users to cancel this command without breaking the test + except KeyboardInterrupt: + pass + + def console_interact(self) -> None: + """ + Allows you to directly interact with QEMU's stdin, by forwarding + terminal input to the QEMU process. + This is for use with the interactive test driver, not for production + tests, which run unattended. + Output from QEMU is only read line-wise. `Ctrl-c` kills QEMU and + `Ctrl-d` closes console and returns to the test runner. + """ + self.log("Terminal is ready (there is no prompt):") + + assert self.process + assert self.process.stdin + + while True: + try: + char = sys.stdin.buffer.read(1) + except KeyboardInterrupt: + break + if char == b"": # ctrl+d + self.log("Closing connection to the console") + break + self.send_console(char.decode()) + + def succeed(self, *commands: str, timeout: Optional[int] = None) -> str: + """ + Execute a shell command, raising an exception if the exit status is + not zero, otherwise returning the standard output. Similar to `execute`, + except that the timeout is `None` by default. See `execute` for details on + command execution. + """ + output = "" + for command in commands: + with self.nested(f"must succeed: {command}"): + (status, out) = self.execute(command, timeout=timeout) + if status != 0: + self.log(f"output: {out}") + raise Exception(f"command `{command}` failed (exit code {status})") + output += out + return output + + def fail(self, *commands: str, timeout: Optional[int] = None) -> str: + """ + Like `succeed`, but raising an exception if the command returns a zero + status. + """ + output = "" + for command in commands: + with self.nested(f"must fail: {command}"): + (status, out) = self.execute(command, timeout=timeout) + if status == 0: + raise Exception(f"command `{command}` unexpectedly succeeded") + output += out + return output + + def wait_until_succeeds(self, command: str, timeout: int = 900) -> str: + """ + Repeat a shell command with 1-second intervals until it succeeds. + Has a default timeout of 900 seconds which can be modified, e.g. + `wait_until_succeeds(cmd, timeout=10)`. See `execute` for details on + command execution. + Throws an exception on timeout. + """ + output = "" + + def check_success(_: Any) -> bool: + nonlocal output + status, output = self.execute(command, timeout=timeout) + return status == 0 + + with self.nested(f"waiting for success: {command}"): + retry(check_success, timeout) + return output + + def wait_until_fails(self, command: str, timeout: int = 900) -> str: + """ + Like `wait_until_succeeds`, but repeating the command until it fails. + """ + output = "" + + def check_failure(_: Any) -> bool: + nonlocal output + status, output = self.execute(command, timeout=timeout) + return status != 0 + + with self.nested(f"waiting for failure: {command}"): + retry(check_failure, timeout) + return output + + def wait_for_shutdown(self) -> None: + if not self.booted: + return + + with self.nested("waiting for the VM to power off"): + sys.stdout.flush() + assert self.process + self.process.wait() + + self.pid = None + self.booted = False + self.connected = False + + def get_tty_text(self, tty: str) -> str: + status, output = self.execute( + f"fold -w$(stty -F /dev/tty{tty} size | " + f"awk '{{print $2}}') /dev/vcs{tty}" + ) + return output + + def wait_until_tty_matches(self, tty: str, regexp: str, timeout: int = 900) -> None: + """Wait until the visible output on the chosen TTY matches regular + expression. Throws an exception on timeout. + """ + matcher = re.compile(regexp) + + def tty_matches(last: bool) -> bool: + text = self.get_tty_text(tty) + if last: + self.log( + f"Last chance to match /{regexp}/ on TTY{tty}, " + f"which currently contains: {text}" + ) + return len(matcher.findall(text)) > 0 + + with self.nested(f"waiting for {regexp} to appear on tty {tty}"): + retry(tty_matches, timeout) + + def send_chars(self, chars: str, delay: Optional[float] = 0.01) -> None: + """ + Simulate typing a sequence of characters on the virtual keyboard, + e.g., `send_chars("foobar\n")` will type the string `foobar` + followed by the Enter key. + """ + with self.nested(f"sending keys {repr(chars)}"): + for char in chars: + self.send_key(char, delay, log=False) + + def wait_for_file(self, filename: str, timeout: int = 900) -> None: + """ + Waits until the file exists in the machine's file system. + """ + + def check_file(_: Any) -> bool: + status, _ = self.execute(f"test -e {filename}") + return status == 0 + + with self.nested(f"waiting for file '{filename}'"): + retry(check_file, timeout) + + def wait_for_open_port( + self, port: int, addr: str = "localhost", timeout: int = 900 + ) -> None: + """ + Wait until a process is listening on the given TCP port and IP address + (default `localhost`). + """ + + def port_is_open(_: Any) -> bool: + status, _ = self.execute(f"nc -z {addr} {port}") + return status == 0 + + with self.nested(f"waiting for TCP port {port} on {addr}"): + retry(port_is_open, timeout) + + def wait_for_open_unix_socket( + self, addr: str, is_datagram: bool = False, timeout: int = 900 + ) -> None: + """ + Wait until a process is listening on the given UNIX-domain socket + (default to a UNIX-domain stream socket). + """ + + nc_flags = [ + "-z", + "-uU" if is_datagram else "-U", + ] + + def socket_is_open(_: Any) -> bool: + status, _ = self.execute(f"nc {' '.join(nc_flags)} {addr}") + return status == 0 + + with self.nested( + f"waiting for UNIX-domain {'datagram' if is_datagram else 'stream'} on '{addr}'" + ): + retry(socket_is_open, timeout) + + def wait_for_closed_port( + self, port: int, addr: str = "localhost", timeout: int = 900 + ) -> None: + """ + Wait until nobody is listening on the given TCP port and IP address + (default `localhost`). + """ + + def port_is_closed(_: Any) -> bool: + status, _ = self.execute(f"nc -z {addr} {port}") + return status != 0 + + with self.nested(f"waiting for TCP port {port} on {addr} to be closed"): + retry(port_is_closed, timeout) + + def start_job(self, jobname: str, user: Optional[str] = None) -> Tuple[int, str]: + return self.systemctl(f"start {jobname}", user) + + def stop_job(self, jobname: str, user: Optional[str] = None) -> Tuple[int, str]: + return self.systemctl(f"stop {jobname}", user) + + def wait_for_job(self, jobname: str) -> None: + self.wait_for_unit(jobname) + + def connect(self) -> None: + def shell_ready(timeout_secs: int) -> bool: + """We sent some data from the backdoor service running on the guest + to indicate that the backdoor shell is ready. + As soon as we read some data from the socket here, we assume that + our root shell is operational. + """ + (ready, _, _) = select.select([self.shell], [], [], timeout_secs) + return bool(ready) + + if self.connected: + return + + with self.nested("waiting for the VM to finish booting"): + self.start() + + assert self.shell + + tic = time.time() + # TODO: do we want to bail after a set number of attempts? + while not shell_ready(timeout_secs=30): + self.log("Guest root shell did not produce any data yet...") + self.log( + " To debug, enter the VM and run 'systemctl status backdoor.service'." + ) + + while True: + chunk = self.shell.recv(1024) + # No need to print empty strings, it means we are waiting. + if len(chunk) == 0: + continue + self.log(f"Guest shell says: {chunk!r}") + # NOTE: for this to work, nothing must be printed after this line! + if b"Spawning backdoor root shell..." in chunk: + break + + toc = time.time() + + self.log("connected to guest root shell") + self.log(f"(connecting took {toc - tic:.2f} seconds)") + self.connected = True + + def screenshot(self, filename: str) -> None: + """ + Take a picture of the display of the virtual machine, in PNG format. + The screenshot will be available in the derivation output. + """ + if "." not in filename: + filename += ".png" + if "/" not in filename: + filename = os.path.join(self.out_dir, filename) + tmp = f"{filename}.ppm" + + with self.nested( + f"making screenshot {filename}", + {"image": os.path.basename(filename)}, + ): + self.send_monitor_command(f"screendump {tmp}") + ret = subprocess.run(f"pnmtopng '{tmp}' > '{filename}'", shell=True) + os.unlink(tmp) + if ret.returncode != 0: + raise Exception("Cannot convert screenshot") + + def copy_from_host_via_shell(self, source: str, target: str) -> None: + """Copy a file from the host into the guest by piping it over the + shell into the destination file. Works without host-guest shared folder. + Prefer copy_from_host for whenever possible. + """ + with open(source, "rb") as fh: + content_b64 = base64.b64encode(fh.read()).decode() + self.succeed( + f"mkdir -p $(dirname {target})", + f"echo -n {content_b64} | base64 -d > {target}", + ) + + def copy_from_host(self, source: str, target: str) -> None: + """ + Copies a file from host to machine, e.g., + `copy_from_host("myfile", "/etc/my/important/file")`. + + The first argument is the file on the host. Note that the "host" refers + to the environment in which the test driver runs, which is typically the + Nix build sandbox. + + The second argument is the location of the file on the machine that will + be written to. + + The file is copied via the `shared_dir` directory which is shared among + all the VMs (using a temporary directory). + The access rights bits will mimic the ones from the host file and + user:group will be root:root. + """ + host_src = Path(source) + vm_target = Path(target) + with tempfile.TemporaryDirectory(dir=self.shared_dir) as shared_td: + shared_temp = Path(shared_td) + host_intermediate = shared_temp / host_src.name + vm_shared_temp = Path("/tmp/shared") / shared_temp.name + vm_intermediate = vm_shared_temp / host_src.name + + self.succeed(make_command(["mkdir", "-p", vm_shared_temp])) + if host_src.is_dir(): + shutil.copytree(host_src, host_intermediate) + else: + shutil.copy(host_src, host_intermediate) + self.succeed(make_command(["mkdir", "-p", vm_target.parent])) + self.succeed(make_command(["cp", "-r", vm_intermediate, vm_target])) + + def copy_from_vm(self, source: str, target_dir: str = "") -> None: + """Copy a file from the VM (specified by an in-VM source path) to a path + relative to `$out`. The file is copied via the `shared_dir` shared among + all the VMs (using a temporary directory). + """ + # Compute the source, target, and intermediate shared file names + vm_src = Path(source) + with tempfile.TemporaryDirectory(dir=self.shared_dir) as shared_td: + shared_temp = Path(shared_td) + vm_shared_temp = Path("/tmp/shared") / shared_temp.name + vm_intermediate = vm_shared_temp / vm_src.name + intermediate = shared_temp / vm_src.name + # Copy the file to the shared directory inside VM + self.succeed(make_command(["mkdir", "-p", vm_shared_temp])) + self.succeed(make_command(["cp", "-r", vm_src, vm_intermediate])) + abs_target = self.out_dir / target_dir / vm_src.name + abs_target.parent.mkdir(exist_ok=True, parents=True) + # Copy the file from the shared directory outside VM + if intermediate.is_dir(): + shutil.copytree(intermediate, abs_target) + else: + shutil.copy(intermediate, abs_target) + + def dump_tty_contents(self, tty: str) -> None: + """Debugging: Dump the contents of the TTY<n>""" + self.execute(f"fold -w 80 /dev/vcs{tty} | systemd-cat") + + def _get_screen_text_variants(self, model_ids: Iterable[int]) -> List[str]: + with tempfile.TemporaryDirectory() as tmpdir: + screenshot_path = os.path.join(tmpdir, "ppm") + self.send_monitor_command(f"screendump {screenshot_path}") + return _perform_ocr_on_screenshot(screenshot_path, model_ids) + + def get_screen_text_variants(self) -> List[str]: + """ + Return a list of different interpretations of what is currently + visible on the machine's screen using optical character + recognition. The number and order of the interpretations is not + specified and is subject to change, but if no exception is raised at + least one will be returned. + + ::: {.note} + This requires [`enableOCR`](#test-opt-enableOCR) to be set to `true`. + ::: + """ + return self._get_screen_text_variants([0, 1, 2]) + + def get_screen_text(self) -> str: + """ + Return a textual representation of what is currently visible on the + machine's screen using optical character recognition. + + ::: {.note} + This requires [`enableOCR`](#test-opt-enableOCR) to be set to `true`. + ::: + """ + return self._get_screen_text_variants([2])[0] + + def wait_for_text(self, regex: str, timeout: int = 900) -> None: + """ + Wait until the supplied regular expressions matches the textual + contents of the screen by using optical character recognition (see + `get_screen_text` and `get_screen_text_variants`). + + ::: {.note} + This requires [`enableOCR`](#test-opt-enableOCR) to be set to `true`. + ::: + """ + + def screen_matches(last: bool) -> bool: + variants = self.get_screen_text_variants() + for text in variants: + if re.search(regex, text) is not None: + return True + + if last: + self.log(f"Last OCR attempt failed. Text was: {variants}") + + return False + + with self.nested(f"waiting for {regex} to appear on screen"): + retry(screen_matches, timeout) + + def wait_for_console_text(self, regex: str, timeout: int | None = None) -> None: + """ + Wait until the supplied regular expressions match a line of the + serial console output. + This method is useful when OCR is not possible or inaccurate. + """ + # Buffer the console output, this is needed + # to match multiline regexes. + console = io.StringIO() + + def console_matches(_: Any) -> bool: + nonlocal console + try: + # This will return as soon as possible and + # sleep 1 second. + console.write(self.last_lines.get(block=False)) + except queue.Empty: + pass + console.seek(0) + matches = re.search(regex, console.read()) + return matches is not None + + with self.nested(f"waiting for {regex} to appear on console"): + if timeout is not None: + retry(console_matches, timeout) + else: + while not console_matches(False): + pass + + def send_key( + self, key: str, delay: Optional[float] = 0.01, log: Optional[bool] = True + ) -> None: + """ + Simulate pressing keys on the virtual keyboard, e.g., + `send_key("ctrl-alt-delete")`. + + Please also refer to the QEMU documentation for more information on the + input syntax: https://en.wikibooks.org/wiki/QEMU/Monitor#sendkey_keys + """ + key = CHAR_TO_KEY.get(key, key) + context = self.nested(f"sending key {repr(key)}") if log else nullcontext() + with context: + self.send_monitor_command(f"sendkey {key}") + if delay is not None: + time.sleep(delay) + + def send_console(self, chars: str) -> None: + r""" + Send keys to the kernel console. This allows interaction with the systemd + emergency mode, for example. Takes a string that is sent, e.g., + `send_console("\n\nsystemctl default\n")`. + """ + assert self.process + assert self.process.stdin + self.process.stdin.write(chars.encode()) + self.process.stdin.flush() + + def start(self, allow_reboot: bool = False) -> None: + """ + Start the virtual machine. This method is asynchronous --- it does + not wait for the machine to finish booting. + """ + if self.booted: + return + + self.log("starting vm") + + def clear(path: Path) -> Path: + if path.exists(): + path.unlink() + return path + + def create_socket(path: Path) -> socket.socket: + s = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM) + s.bind(str(path)) + s.listen(1) + return s + + monitor_socket = create_socket(clear(self.monitor_path)) + shell_socket = create_socket(clear(self.shell_path)) + self.process = self.start_command.run( + self.state_dir, + self.shared_dir, + self.monitor_path, + self.qmp_path, + self.shell_path, + allow_reboot, + ) + self.monitor, _ = monitor_socket.accept() + self.shell, _ = shell_socket.accept() + self.qmp_client = QMPSession.from_path(self.qmp_path) + + # Store last serial console lines for use + # of wait_for_console_text + self.last_lines: Queue = Queue() + + def process_serial_output() -> None: + assert self.process + assert self.process.stdout + for _line in self.process.stdout: + # Ignore undecodable bytes that may occur in boot menus + line = _line.decode(errors="ignore").replace("\r", "").rstrip() + self.last_lines.put(line) + self.log_serial(line) + + self.serial_thread = threading.Thread(target=process_serial_output) + self.serial_thread.start() + + self.wait_for_monitor_prompt() + + self.pid = self.process.pid + self.booted = True + + self.log(f"QEMU running (pid {self.pid})") + + def cleanup_statedir(self) -> None: + shutil.rmtree(self.state_dir) + rootlog.log(f"deleting VM state directory {self.state_dir}") + rootlog.log("if you want to keep the VM state, pass --keep-vm-state") + + def shutdown(self) -> None: + """ + Shut down the machine, waiting for the VM to exit. + """ + if not self.booted: + return + + assert self.shell + self.shell.send(b"poweroff\n") + self.wait_for_shutdown() + + def crash(self) -> None: + """ + Simulate a sudden power failure, by telling the VM to exit immediately. + """ + if not self.booted: + return + + self.log("forced crash") + self.send_monitor_command("quit") + self.wait_for_shutdown() + + def reboot(self) -> None: + """Press Ctrl+Alt+Delete in the guest. + + Prepares the machine to be reconnected which is useful if the + machine was started with `allow_reboot = True` + """ + self.send_key("ctrl-alt-delete") + self.connected = False + + def wait_for_x(self, timeout: int = 900) -> None: + """ + Wait until it is possible to connect to the X server. + """ + + def check_x(_: Any) -> bool: + cmd = ( + "journalctl -b SYSLOG_IDENTIFIER=systemd | " + + 'grep "Reached target Current graphical"' + ) + status, _ = self.execute(cmd) + if status != 0: + return False + status, _ = self.execute("[ -e /tmp/.X11-unix/X0 ]") + return status == 0 + + with self.nested("waiting for the X11 server"): + retry(check_x, timeout) + + def get_window_names(self) -> List[str]: + return self.succeed( + r"xwininfo -root -tree | sed 's/.*0x[0-9a-f]* \"\([^\"]*\)\".*/\1/; t; d'" + ).splitlines() + + def wait_for_window(self, regexp: str, timeout: int = 900) -> None: + """ + Wait until an X11 window has appeared whose name matches the given + regular expression, e.g., `wait_for_window("Terminal")`. + """ + pattern = re.compile(regexp) + + def window_is_visible(last_try: bool) -> bool: + names = self.get_window_names() + if last_try: + self.log( + f"Last chance to match {regexp} on the window list," + + " which currently contains: " + + ", ".join(names) + ) + return any(pattern.search(name) for name in names) + + with self.nested("waiting for a window to appear"): + retry(window_is_visible, timeout) + + def sleep(self, secs: int) -> None: + # We want to sleep in *guest* time, not *host* time. + self.succeed(f"sleep {secs}") + + def forward_port(self, host_port: int = 8080, guest_port: int = 80) -> None: + """ + Forward a TCP port on the host to a TCP port on the guest. + Useful during interactive testing. + """ + self.send_monitor_command(f"hostfwd_add tcp::{host_port}-:{guest_port}") + + def block(self) -> None: + """ + Simulate unplugging the Ethernet cable that connects the machine to + the other machines. + This happens by shutting down eth1 (the multicast interface used to talk + to the other VMs). eth0 is kept online to still enable the test driver + to communicate with the machine. + """ + self.send_monitor_command("set_link virtio-net-pci.1 off") + + def unblock(self) -> None: + """ + Undo the effect of `block`. + """ + self.send_monitor_command("set_link virtio-net-pci.1 on") + + def release(self) -> None: + if self.pid is None: + return + rootlog.info(f"kill machine (pid {self.pid})") + assert self.process + assert self.shell + assert self.monitor + assert self.serial_thread + + self.process.terminate() + self.shell.close() + self.monitor.close() + self.serial_thread.join() + + def run_callbacks(self) -> None: + for callback in self.callbacks: + callback() + + def switch_root(self) -> None: + """ + Transition from stage 1 to stage 2. This requires the + machine to be configured with `testing.initrdBackdoor = true` + and `boot.initrd.systemd.enable = true`. + """ + self.wait_for_unit("initrd.target") + self.execute( + "systemctl isolate --no-block initrd-switch-root.target 2>/dev/null >/dev/null", + check_return=False, + check_output=False, + ) + self.wait_for_console_text(r"systemd\[1\]:.*Switching root\.") + self.connected = False + self.connect() diff --git a/nixpkgs/nixos/lib/test-driver/test_driver/polling_condition.py b/nixpkgs/nixos/lib/test-driver/test_driver/polling_condition.py new file mode 100644 index 000000000000..12cbad69e34e --- /dev/null +++ b/nixpkgs/nixos/lib/test-driver/test_driver/polling_condition.py @@ -0,0 +1,92 @@ +import time +from math import isfinite +from typing import Callable, Optional + +from .logger import rootlog + + +class PollingConditionError(Exception): + pass + + +class PollingCondition: + condition: Callable[[], bool] + seconds_interval: float + description: Optional[str] + + last_called: float + entry_count: int + + def __init__( + self, + condition: Callable[[], Optional[bool]], + seconds_interval: float = 2.0, + description: Optional[str] = None, + ): + self.condition = condition # type: ignore + self.seconds_interval = seconds_interval + + if description is None: + if condition.__doc__: + self.description = condition.__doc__ + else: + self.description = condition.__name__ + else: + self.description = str(description) + + self.last_called = float("-inf") + self.entry_count = 0 + + def check(self, force: bool = False) -> bool: + if (self.entered or not self.overdue) and not force: + return True + + with self, rootlog.nested(self.nested_message): + time_since_last = time.monotonic() - self.last_called + last_message = ( + f"Time since last: {time_since_last:.2f}s" + if isfinite(time_since_last) + else "(not called yet)" + ) + + rootlog.info(last_message) + try: + res = self.condition() # type: ignore + except Exception: + res = False + res = res is None or res + rootlog.info(self.status_message(res)) + return res + + def maybe_raise(self) -> None: + if not self.check(): + raise PollingConditionError(self.status_message(False)) + + def status_message(self, status: bool) -> str: + return f"Polling condition {'succeeded' if status else 'failed'}: {self.description}" + + @property + def nested_message(self) -> str: + nested_message = ["Checking polling condition"] + if self.description is not None: + nested_message.append(repr(self.description)) + + return " ".join(nested_message) + + @property + def overdue(self) -> bool: + return self.last_called + self.seconds_interval < time.monotonic() + + @property + def entered(self) -> bool: + # entry_count should never dip *below* zero + assert self.entry_count >= 0 + return self.entry_count > 0 + + def __enter__(self) -> None: + self.entry_count += 1 + + def __exit__(self, exc_type, exc_value, traceback) -> None: # type: ignore + assert self.entered + self.entry_count -= 1 + self.last_called = time.monotonic() diff --git a/nixpkgs/nixos/lib/test-driver/test_driver/py.typed b/nixpkgs/nixos/lib/test-driver/test_driver/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/nixpkgs/nixos/lib/test-driver/test_driver/py.typed diff --git a/nixpkgs/nixos/lib/test-driver/test_driver/qmp.py b/nixpkgs/nixos/lib/test-driver/test_driver/qmp.py new file mode 100644 index 000000000000..62ca6d7d5b80 --- /dev/null +++ b/nixpkgs/nixos/lib/test-driver/test_driver/qmp.py @@ -0,0 +1,98 @@ +import json +import logging +import os +import socket +from collections.abc import Iterator +from pathlib import Path +from queue import Queue +from typing import Any + +logger = logging.getLogger(__name__) + + +class QMPAPIError(RuntimeError): + def __init__(self, message: dict[str, Any]): + assert "error" in message, "Not an error message!" + try: + self.class_name = message["class"] + self.description = message["desc"] + # NOTE: Some errors can occur before the Server is able to read the + # id member; in these cases the id member will not be part of the + # error response, even if provided by the client. + self.transaction_id = message.get("id") + except KeyError: + raise RuntimeError("Malformed QMP API error response") + + def __str__(self) -> str: + return f"<QMP API error related to transaction {self.transaction_id} [{self.class_name}]: {self.description}>" + + +class QMPSession: + def __init__(self, sock: socket.socket) -> None: + self.sock = sock + self.results: Queue[dict[str, str]] = Queue() + self.pending_events: Queue[dict[str, Any]] = Queue() + self.reader = sock.makefile("r") + self.writer = sock.makefile("w") + # Make the reader non-blocking so we can kind of select on it. + os.set_blocking(self.reader.fileno(), False) + hello = self._wait_for_new_result() + logger.debug(f"Got greeting from QMP API: {hello}") + # The greeting message format is: + # { "QMP": { "version": json-object, "capabilities": json-array } } + assert "QMP" in hello, f"Unexpected result: {hello}" + self.send("qmp_capabilities") + + @classmethod + def from_path(cls, path: Path) -> "QMPSession": + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(str(path)) + return cls(sock) + + def __del__(self) -> None: + self.sock.close() + + def _wait_for_new_result(self) -> dict[str, str]: + assert self.results.empty(), "Results set is not empty, missed results!" + while self.results.empty(): + self.read_pending_messages() + return self.results.get() + + def read_pending_messages(self) -> None: + line = self.reader.readline() + if not line: + return + evt_or_result = json.loads(line) + logger.debug(f"Received a message: {evt_or_result}") + + # It's a result + if "return" in evt_or_result or "QMP" in evt_or_result: + self.results.put(evt_or_result) + # It's an event + elif "event" in evt_or_result: + self.pending_events.put(evt_or_result) + else: + raise QMPAPIError(evt_or_result) + + def wait_for_event(self, timeout: int = 10) -> dict[str, Any]: + while self.pending_events.empty(): + self.read_pending_messages() + + return self.pending_events.get(timeout=timeout) + + def events(self, timeout: int = 10) -> Iterator[dict[str, Any]]: + while not self.pending_events.empty(): + yield self.pending_events.get(timeout=timeout) + + def send(self, cmd: str, args: dict[str, str] = {}) -> dict[str, str]: + self.read_pending_messages() + assert self.results.empty(), "Results set is not empty, missed results!" + data: dict[str, Any] = dict(execute=cmd) + if args != {}: + data["arguments"] = args + + logger.debug(f"Sending {data} to QMP...") + json.dump(data, self.writer) + self.writer.write("\n") + self.writer.flush() + return self._wait_for_new_result() diff --git a/nixpkgs/nixos/lib/test-driver/test_driver/vlan.py b/nixpkgs/nixos/lib/test-driver/test_driver/vlan.py new file mode 100644 index 000000000000..ec9679108e58 --- /dev/null +++ b/nixpkgs/nixos/lib/test-driver/test_driver/vlan.py @@ -0,0 +1,62 @@ +import io +import os +import pty +import subprocess +from pathlib import Path + +from test_driver.logger import rootlog + + +class VLan: + """This class handles a VLAN that the run-vm scripts identify via its + number handles. The network's lifetime equals the object's lifetime. + """ + + nr: int + socket_dir: Path + + process: subprocess.Popen + pid: int + fd: io.TextIOBase + + def __repr__(self) -> str: + return f"<Vlan Nr. {self.nr}>" + + def __init__(self, nr: int, tmp_dir: Path): + self.nr = nr + self.socket_dir = tmp_dir / f"vde{self.nr}.ctl" + + # TODO: don't side-effect environment here + os.environ[f"QEMU_VDE_SOCKET_{self.nr}"] = str(self.socket_dir) + + rootlog.info("start vlan") + pty_master, pty_slave = pty.openpty() + + # The --hub is required for the scenario determined by + # nixos/tests/networking.nix vlan-ping. + # VLAN Tagged traffic (802.1Q) seams to be blocked if a vde_switch is + # used without the hub mode (flood packets to all ports). + self.process = subprocess.Popen( + ["vde_switch", "-s", self.socket_dir, "--dirmode", "0700", "--hub"], + stdin=pty_slave, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + ) + self.pid = self.process.pid + self.fd = os.fdopen(pty_master, "w") + self.fd.write("version\n") + + # TODO: perl version checks if this can be read from + # an if not, dies. we could hang here forever. Fix it. + assert self.process.stdout is not None + self.process.stdout.readline() + if not (self.socket_dir / "ctl").exists(): + rootlog.error("cannot start vde_switch") + + rootlog.info(f"running vlan (pid {self.pid}; ctl {self.socket_dir})") + + def __del__(self) -> None: + rootlog.info(f"kill vlan (pid {self.pid})") + self.fd.close() + self.process.terminate() diff --git a/nixpkgs/nixos/lib/test-script-prepend.py b/nixpkgs/nixos/lib/test-script-prepend.py new file mode 100644 index 000000000000..15e59ce01047 --- /dev/null +++ b/nixpkgs/nixos/lib/test-script-prepend.py @@ -0,0 +1,42 @@ +# This file contains type hints that can be prepended to Nix test scripts so they can be type +# checked. + +from test_driver.driver import Driver +from test_driver.vlan import VLan +from test_driver.machine import Machine +from test_driver.logger import Logger +from typing import Callable, Iterator, ContextManager, Optional, List, Dict, Any, Union +from typing_extensions import Protocol +from pathlib import Path + + +class RetryProtocol(Protocol): + def __call__(self, fn: Callable, timeout: int = 900) -> None: + raise Exception("This is just type information for the Nix test driver") + + +class PollingConditionProtocol(Protocol): + def __call__( + self, + fun_: Optional[Callable] = None, + *, + seconds_interval: float = 2.0, + description: Optional[str] = None, + ) -> Union[Callable[[Callable], ContextManager], ContextManager]: + raise Exception("This is just type information for the Nix test driver") + + +start_all: Callable[[], None] +subtest: Callable[[str], ContextManager[None]] +retry: RetryProtocol +test_script: Callable[[], None] +machines: List[Machine] +vlans: List[VLan] +driver: Driver +log: Logger +create_machine: Callable[[Dict[str, Any]], Machine] +run_tests: Callable[[], None] +join_all: Callable[[], None] +serial_stdout_off: Callable[[], None] +serial_stdout_on: Callable[[], None] +polling_condition: PollingConditionProtocol diff --git a/nixpkgs/nixos/lib/testing-python.nix b/nixpkgs/nixos/lib/testing-python.nix new file mode 100644 index 000000000000..f5222351518b --- /dev/null +++ b/nixpkgs/nixos/lib/testing-python.nix @@ -0,0 +1,78 @@ +args@ +{ system +, pkgs ? import ../.. { inherit system config; } + # Use a minimal kernel? +, minimal ? false + # Ignored +, config ? { } + # !!! See comment about args in lib/modules.nix +, specialArgs ? throw "legacy - do not use, see error below" + # Modules to add to each VM +, extraConfigurations ? [ ] +}: +let + nixos-lib = import ./default.nix { inherit (pkgs) lib; }; +in + +pkgs.lib.throwIf (args?specialArgs) '' + testing-python.nix: `specialArgs` is not supported anymore. If you're looking + for the public interface to the NixOS test framework, use `runTest`, and + `node.specialArgs`. + See https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests + and https://nixos.org/manual/nixos/unstable/index.html#test-opt-node.specialArgs +'' +rec { + + inherit pkgs; + + evalTest = module: nixos-lib.evalTest { imports = [ extraTestModule module ]; }; + runTest = module: nixos-lib.runTest { imports = [ extraTestModule module ]; }; + + extraTestModule = { + config = { + hostPkgs = pkgs; + }; + }; + + # Make a full-blown test (legacy) + # For an official public interface to the tests, see + # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests + makeTest = + { machine ? null + , nodes ? {} + , testScript + , enableOCR ? false + , globalTimeout ? (60 * 60) + , name ? "unnamed" + , skipTypeCheck ? false + # Skip linting (mainly intended for faster dev cycles) + , skipLint ? false + , passthru ? {} + , meta ? {} + , # For meta.position + pos ? # position used in error messages and for meta.position + (if meta.description or null != null + then builtins.unsafeGetAttrPos "description" meta + else builtins.unsafeGetAttrPos "testScript" t) + , extraPythonPackages ? (_ : []) + , interactive ? {} + } @ t: let + testConfig = + (evalTest { + imports = [ + { _file = "makeTest parameters"; config = t; } + { + defaults = { + _file = "makeTest: extraConfigurations"; + imports = extraConfigurations; + }; + } + ]; + }).config; + in + testConfig.test # For nix-build + // testConfig; # For all-tests.nix + + simpleTest = as: (makeTest as).test; + +} diff --git a/nixpkgs/nixos/lib/testing/call-test.nix b/nixpkgs/nixos/lib/testing/call-test.nix new file mode 100644 index 000000000000..9abcea07455e --- /dev/null +++ b/nixpkgs/nixos/lib/testing/call-test.nix @@ -0,0 +1,12 @@ +{ config, lib, ... }: +let + inherit (lib) mkOption types; +in +{ + options = { + result = mkOption { + internal = true; + default = config; + }; + }; +} diff --git a/nixpkgs/nixos/lib/testing/default.nix b/nixpkgs/nixos/lib/testing/default.nix new file mode 100644 index 000000000000..a89f734b1e64 --- /dev/null +++ b/nixpkgs/nixos/lib/testing/default.nix @@ -0,0 +1,27 @@ +{ lib }: +let + + evalTest = module: lib.evalModules { + modules = testModules ++ [ module ]; + class = "nixosTest"; + }; + runTest = module: (evalTest ({ config, ... }: { imports = [ module ]; result = config.test; })).config.result; + + testModules = [ + ./call-test.nix + ./driver.nix + ./interactive.nix + ./legacy.nix + ./meta.nix + ./name.nix + ./network.nix + ./nodes.nix + ./pkgs.nix + ./run.nix + ./testScript.nix + ]; + +in +{ + inherit evalTest runTest testModules; +} diff --git a/nixpkgs/nixos/lib/testing/driver.nix b/nixpkgs/nixos/lib/testing/driver.nix new file mode 100644 index 000000000000..b6f01c38191d --- /dev/null +++ b/nixpkgs/nixos/lib/testing/driver.nix @@ -0,0 +1,203 @@ +{ config, lib, hostPkgs, ... }: +let + inherit (lib) mkOption types literalMD mdDoc; + + # Reifies and correctly wraps the python test driver for + # the respective qemu version and with or without ocr support + testDriver = hostPkgs.callPackage ../test-driver { + inherit (config) enableOCR extraPythonPackages; + qemu_pkg = config.qemu.package; + imagemagick_light = hostPkgs.imagemagick_light.override { inherit (hostPkgs) libtiff; }; + tesseract4 = hostPkgs.tesseract4.override { enableLanguages = [ "eng" ]; }; + }; + + + vlans = map (m: ( + m.virtualisation.vlans ++ + (lib.mapAttrsToList (_: v: v.vlan) m.virtualisation.interfaces))) (lib.attrValues config.nodes); + vms = map (m: m.system.build.vm) (lib.attrValues config.nodes); + + nodeHostNames = + let + nodesList = map (c: c.system.name) (lib.attrValues config.nodes); + in + nodesList ++ lib.optional (lib.length nodesList == 1 && !lib.elem "machine" nodesList) "machine"; + + pythonizeName = name: + let + head = lib.substring 0 1 name; + tail = lib.substring 1 (-1) name; + in + (if builtins.match "[A-z_]" head == null then "_" else head) + + lib.stringAsChars (c: if builtins.match "[A-z0-9_]" c == null then "_" else c) tail; + + uniqueVlans = lib.unique (builtins.concatLists vlans); + vlanNames = map (i: "vlan${toString i}: VLan;") uniqueVlans; + pythonizedNames = map pythonizeName nodeHostNames; + machineNames = map (name: "${name}: Machine;") pythonizedNames; + + withChecks = lib.warnIf config.skipLint "Linting is disabled"; + + driver = + hostPkgs.runCommand "nixos-test-driver-${config.name}" + { + # inherit testName; TODO (roberth): need this? + nativeBuildInputs = [ + hostPkgs.makeWrapper + ] ++ lib.optionals (!config.skipTypeCheck) [ hostPkgs.mypy ]; + buildInputs = [ testDriver ]; + testScript = config.testScriptString; + preferLocalBuild = true; + passthru = config.passthru; + meta = config.meta // { + mainProgram = "nixos-test-driver"; + }; + } + '' + mkdir -p $out/bin + + vmStartScripts=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done)) + + ${lib.optionalString (!config.skipTypeCheck) '' + # prepend type hints so the test script can be type checked with mypy + cat "${../test-script-prepend.py}" >> testScriptWithTypes + echo "${builtins.toString machineNames}" >> testScriptWithTypes + echo "${builtins.toString vlanNames}" >> testScriptWithTypes + echo -n "$testScript" >> testScriptWithTypes + + echo "Running type check (enable/disable: config.skipTypeCheck)" + echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipTypeCheck" + + mypy --no-implicit-optional \ + --pretty \ + --no-color-output \ + testScriptWithTypes + ''} + + echo -n "$testScript" >> $out/test-script + + ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver + + ${testDriver}/bin/generate-driver-symbols + ${lib.optionalString (!config.skipLint) '' + echo "Linting test script (enable/disable: config.skipLint)" + echo "See https://nixos.org/manual/nixos/stable/#test-opt-skipLint" + + PYFLAKES_BUILTINS="$( + echo -n ${lib.escapeShellArg (lib.concatStringsSep "," pythonizedNames)}, + < ${lib.escapeShellArg "driver-symbols"} + )" ${hostPkgs.python3Packages.pyflakes}/bin/pyflakes $out/test-script + ''} + + # set defaults through environment + # see: ./test-driver/test-driver.py argparse implementation + wrapProgram $out/bin/nixos-test-driver \ + --set startScripts "''${vmStartScripts[*]}" \ + --set testScript "$out/test-script" \ + --set globalTimeout "${toString config.globalTimeout}" \ + --set vlans '${toString vlans}' \ + ${lib.escapeShellArgs (lib.concatMap (arg: ["--add-flags" arg]) config.extraDriverArgs)} + ''; + +in +{ + options = { + + driver = mkOption { + description = mdDoc "Package containing a script that runs the test."; + type = types.package; + defaultText = literalMD "set by the test framework"; + }; + + hostPkgs = mkOption { + description = mdDoc "Nixpkgs attrset used outside the nodes."; + type = types.raw; + example = lib.literalExpression '' + import nixpkgs { inherit system config overlays; } + ''; + }; + + qemu.package = mkOption { + description = mdDoc "Which qemu package to use for the virtualisation of [{option}`nodes`](#test-opt-nodes)."; + type = types.package; + default = hostPkgs.qemu_test; + defaultText = "hostPkgs.qemu_test"; + }; + + globalTimeout = mkOption { + description = mdDoc '' + A global timeout for the complete test, expressed in seconds. + Beyond that timeout, every resource will be killed and released and the test will fail. + + By default, we use a 1 hour timeout. + ''; + type = types.int; + default = 60 * 60; + example = 10 * 60; + }; + + enableOCR = mkOption { + description = mdDoc '' + Whether to enable Optical Character Recognition functionality for + testing graphical programs. See [Machine objects](`ssec-machine-objects`). + ''; + type = types.bool; + default = false; + }; + + extraPythonPackages = mkOption { + description = mdDoc '' + Python packages to add to the test driver. + + The argument is a Python package set, similar to `pkgs.pythonPackages`. + ''; + example = lib.literalExpression '' + p: [ p.numpy ] + ''; + type = types.functionTo (types.listOf types.package); + default = ps: [ ]; + }; + + extraDriverArgs = mkOption { + description = mdDoc '' + Extra arguments to pass to the test driver. + + They become part of [{option}`driver`](#test-opt-driver) via `wrapProgram`. + ''; + type = types.listOf types.str; + default = []; + }; + + skipLint = mkOption { + type = types.bool; + default = false; + description = mdDoc '' + Do not run the linters. This may speed up your iteration cycle, but it is not something you should commit. + ''; + }; + + skipTypeCheck = mkOption { + type = types.bool; + default = false; + description = mdDoc '' + Disable type checking. This must not be enabled for new NixOS tests. + + This may speed up your iteration cycle, unless you're working on the [{option}`testScript`](#test-opt-testScript). + ''; + }; + }; + + config = { + _module.args = { + hostPkgs = + # Comment is in nixos/modules/misc/nixpkgs.nix + lib.mkOverride lib.modules.defaultOverridePriority + config.hostPkgs.__splicedPackages; + }; + + driver = withChecks driver; + + # make available on the test runner + passthru.driver = config.driver; + }; +} diff --git a/nixpkgs/nixos/lib/testing/interactive.nix b/nixpkgs/nixos/lib/testing/interactive.nix new file mode 100644 index 000000000000..317ed4241882 --- /dev/null +++ b/nixpkgs/nixos/lib/testing/interactive.nix @@ -0,0 +1,45 @@ +{ config, lib, moduleType, hostPkgs, ... }: +let + inherit (lib) mkOption types mdDoc; +in +{ + options = { + interactive = mkOption { + description = mdDoc '' + Tests [can be run interactively](#sec-running-nixos-tests-interactively) + using the program in the test derivation's `.driverInteractive` attribute. + + When they are, the configuration will include anything set in this submodule. + + You can set any top-level test option here. + + Example test module: + + ```nix + { config, lib, ... }: { + + nodes.rabbitmq = { + services.rabbitmq.enable = true; + }; + + # When running interactively ... + interactive.nodes.rabbitmq = { + # ... enable the web ui. + services.rabbitmq.managementPlugin.enable = true; + }; + } + ``` + + For details, see the section about [running tests interactively](#sec-running-nixos-tests-interactively). + ''; + type = moduleType; + visible = "shallow"; + }; + }; + + config = { + interactive.qemu.package = hostPkgs.qemu; + interactive.extraDriverArgs = [ "--interactive" ]; + passthru.driverInteractive = config.interactive.driver; + }; +} diff --git a/nixpkgs/nixos/lib/testing/legacy.nix b/nixpkgs/nixos/lib/testing/legacy.nix new file mode 100644 index 000000000000..b31057556601 --- /dev/null +++ b/nixpkgs/nixos/lib/testing/legacy.nix @@ -0,0 +1,26 @@ +{ config, options, lib, ... }: +let + inherit (lib) mkIf mkOption types; +in +{ + # This needs options.warnings and options.assertions, which we don't have (yet?). + # imports = [ + # (lib.mkRenamedOptionModule [ "machine" ] [ "nodes" "machine" ]) + # (lib.mkRemovedOptionModule [ "minimal" ] "The minimal kernel module was removed as it was broken and not used any more in nixpkgs.") + # ]; + + options = { + machine = mkOption { + internal = true; + type = types.raw; + }; + }; + + config = { + nodes = mkIf options.machine.isDefined ( + lib.warn + "In test `${config.name}': The `machine' attribute in NixOS tests (pkgs.nixosTest / make-test-python.nix / testing-python.nix / makeTest) is deprecated. Please set the equivalent `nodes.machine'." + { inherit (config) machine; } + ); + }; +} diff --git a/nixpkgs/nixos/lib/testing/meta.nix b/nixpkgs/nixos/lib/testing/meta.nix new file mode 100644 index 000000000000..805b7520edff --- /dev/null +++ b/nixpkgs/nixos/lib/testing/meta.nix @@ -0,0 +1,42 @@ +{ lib, ... }: +let + inherit (lib) types mkOption mdDoc; +in +{ + options = { + meta = lib.mkOption { + description = mdDoc '' + The [`meta`](https://nixos.org/manual/nixpkgs/stable/#chap-meta) attributes that will be set on the returned derivations. + + Not all [`meta`](https://nixos.org/manual/nixpkgs/stable/#chap-meta) attributes are supported, but more can be added as desired. + ''; + apply = lib.filterAttrs (k: v: v != null); + type = types.submodule { + options = { + maintainers = lib.mkOption { + type = types.listOf types.raw; + default = []; + description = mdDoc '' + The [list of maintainers](https://nixos.org/manual/nixpkgs/stable/#var-meta-maintainers) for this test. + ''; + }; + timeout = lib.mkOption { + type = types.nullOr types.int; + default = 3600; # 1 hour + description = mdDoc '' + The [{option}`test`](#test-opt-test)'s [`meta.timeout`](https://nixos.org/manual/nixpkgs/stable/#var-meta-timeout) in seconds. + ''; + }; + broken = lib.mkOption { + type = types.bool; + default = false; + description = mdDoc '' + Sets the [`meta.broken`](https://nixos.org/manual/nixpkgs/stable/#var-meta-broken) attribute on the [{option}`test`](#test-opt-test) derivation. + ''; + }; + }; + }; + default = {}; + }; + }; +} diff --git a/nixpkgs/nixos/lib/testing/name.nix b/nixpkgs/nixos/lib/testing/name.nix new file mode 100644 index 000000000000..0af593169eec --- /dev/null +++ b/nixpkgs/nixos/lib/testing/name.nix @@ -0,0 +1,14 @@ +{ lib, ... }: +let + inherit (lib) mkOption types mdDoc; +in +{ + options.name = mkOption { + description = mdDoc '' + The name of the test. + + This is used in the derivation names of the [{option}`driver`](#test-opt-driver) and [{option}`test`](#test-opt-test) runner. + ''; + type = types.str; + }; +} diff --git a/nixpkgs/nixos/lib/testing/network.nix b/nixpkgs/nixos/lib/testing/network.nix new file mode 100644 index 000000000000..1edc9e276530 --- /dev/null +++ b/nixpkgs/nixos/lib/testing/network.nix @@ -0,0 +1,131 @@ +{ lib, nodes, ... }: + +let + inherit (lib) + attrNames concatMap concatMapStrings flip forEach head + listToAttrs mkDefault mkOption nameValuePair optionalString + range toLower types zipListsWith zipLists + mdDoc + ; + + nodeNumbers = + listToAttrs + (zipListsWith + nameValuePair + (attrNames nodes) + (range 1 254) + ); + + networkModule = { config, nodes, pkgs, ... }: + let + qemu-common = import ../qemu-common.nix { inherit lib pkgs; }; + + # Convert legacy VLANs to named interfaces and merge with explicit interfaces. + vlansNumbered = forEach (zipLists config.virtualisation.vlans (range 1 255)) (v: { + name = "eth${toString v.snd}"; + vlan = v.fst; + assignIP = true; + }); + explicitInterfaces = lib.mapAttrsToList (n: v: v // { name = n; }) config.virtualisation.interfaces; + interfaces = vlansNumbered ++ explicitInterfaces; + interfacesNumbered = zipLists interfaces (range 1 255); + + # Automatically assign IP addresses to requested interfaces. + assignIPs = lib.filter (i: i.assignIP) interfaces; + ipInterfaces = forEach assignIPs (i: + nameValuePair i.name { ipv4.addresses = + [ { address = "192.168.${toString i.vlan}.${toString config.virtualisation.test.nodeNumber}"; + prefixLength = 24; + }]; + }); + + qemuOptions = lib.flatten (forEach interfacesNumbered ({ fst, snd }: + qemu-common.qemuNICFlags snd fst.vlan config.virtualisation.test.nodeNumber)); + udevRules = forEach interfacesNumbered ({ fst, snd }: + # MAC Addresses for QEMU network devices are lowercase, and udev string comparison is case-sensitive. + ''SUBSYSTEM=="net",ACTION=="add",ATTR{address}=="${toLower(qemu-common.qemuNicMac fst.vlan config.virtualisation.test.nodeNumber)}",NAME="${fst.name}"''); + + networkConfig = + { + networking.hostName = mkDefault config.virtualisation.test.nodeName; + + networking.interfaces = listToAttrs ipInterfaces; + + networking.primaryIPAddress = + optionalString (ipInterfaces != [ ]) (head (head ipInterfaces).value.ipv4.addresses).address; + + # Put the IP addresses of all VMs in this machine's + # /etc/hosts file. If a machine has multiple + # interfaces, use the IP address corresponding to + # the first interface (i.e. the first network in its + # virtualisation.vlans option). + networking.extraHosts = flip concatMapStrings (attrNames nodes) + (m': + let config = nodes.${m'}; in + optionalString (config.networking.primaryIPAddress != "") + ("${config.networking.primaryIPAddress} " + + optionalString (config.networking.domain != null) + "${config.networking.hostName}.${config.networking.domain} " + + "${config.networking.hostName}\n")); + + virtualisation.qemu.options = qemuOptions; + boot.initrd.services.udev.rules = concatMapStrings (x: x + "\n") udevRules; + }; + + in + { + key = "network-interfaces"; + config = networkConfig // { + # Expose the networkConfig items for tests like nixops + # that need to recreate the network config. + system.build.networkConfig = networkConfig; + }; + }; + + nodeNumberModule = (regular@{ config, name, ... }: { + options = { + virtualisation.test.nodeName = mkOption { + internal = true; + default = name; + # We need to force this in specilisations, otherwise it'd be + # readOnly = true; + description = mdDoc '' + The `name` in `nodes.<name>`; stable across `specialisations`. + ''; + }; + virtualisation.test.nodeNumber = mkOption { + internal = true; + type = types.int; + readOnly = true; + default = nodeNumbers.${config.virtualisation.test.nodeName}; + description = mdDoc '' + A unique number assigned for each node in `nodes`. + ''; + }; + + # specialisations override the `name` module argument, + # so we push the real `virtualisation.test.nodeName`. + specialisation = mkOption { + type = types.attrsOf (types.submodule { + options.configuration = mkOption { + type = types.submoduleWith { + modules = [ + { + config.virtualisation.test.nodeName = + # assert regular.config.virtualisation.test.nodeName != "configuration"; + regular.config.virtualisation.test.nodeName; + } + ]; + }; + }; + }); + }; + }; + }); + +in +{ + config = { + extraBaseModules = { imports = [ networkModule nodeNumberModule ]; }; + }; +} diff --git a/nixpkgs/nixos/lib/testing/nixos-test-base.nix b/nixpkgs/nixos/lib/testing/nixos-test-base.nix new file mode 100644 index 000000000000..59e6e3843367 --- /dev/null +++ b/nixpkgs/nixos/lib/testing/nixos-test-base.nix @@ -0,0 +1,23 @@ +# A module containing the base imports and overrides that +# are always applied in NixOS VM tests, unconditionally, +# even in `inheritParentConfig = false` specialisations. +{ lib, ... }: +let + inherit (lib) mkForce; +in +{ + imports = [ + ../../modules/virtualisation/qemu-vm.nix + ../../modules/testing/test-instrumentation.nix # !!! should only get added for automated test runs + { key = "no-manual"; documentation.nixos.enable = false; } + { + key = "no-revision"; + # Make the revision metadata constant, in order to avoid needless retesting. + # The human version (e.g. 21.05-pre) is left as is, because it is useful + # for external modules that test with e.g. testers.nixosTest and rely on that + # version number. + config.system.nixos.revision = mkForce "constant-nixos-revision"; + } + + ]; +} diff --git a/nixpkgs/nixos/lib/testing/nodes.nix b/nixpkgs/nixos/lib/testing/nodes.nix new file mode 100644 index 000000000000..73e6d386fd1d --- /dev/null +++ b/nixpkgs/nixos/lib/testing/nodes.nix @@ -0,0 +1,149 @@ +testModuleArgs@{ config, lib, hostPkgs, nodes, ... }: + +let + inherit (lib) + literalExpression + literalMD + mapAttrs + mdDoc + mkDefault + mkIf + mkOption mkForce + optional + optionalAttrs + types + ; + + baseOS = + import ../eval-config.nix { + inherit lib; + system = null; # use modularly defined system + inherit (config.node) specialArgs; + modules = [ config.defaults ]; + baseModules = (import ../../modules/module-list.nix) ++ + [ + ./nixos-test-base.nix + { key = "nodes"; _module.args.nodes = config.nodesCompat; } + ({ config, ... }: + { + virtualisation.qemu.package = testModuleArgs.config.qemu.package; + }) + ({ options, ... }: { + key = "nodes.nix-pkgs"; + config = optionalAttrs (!config.node.pkgsReadOnly) ( + mkIf (!options.nixpkgs.pkgs.isDefined) { + # TODO: switch to nixpkgs.hostPlatform and make sure containers-imperative test still evaluates. + nixpkgs.system = hostPkgs.stdenv.hostPlatform.system; + } + ); + }) + testModuleArgs.config.extraBaseModules + ]; + }; + + +in + +{ + + options = { + node.type = mkOption { + type = types.raw; + default = baseOS.type; + internal = true; + }; + + nodes = mkOption { + type = types.lazyAttrsOf config.node.type; + visible = "shallow"; + description = mdDoc '' + An attribute set of NixOS configuration modules. + + The configurations are augmented by the [`defaults`](#test-opt-defaults) option. + + They are assigned network addresses according to the `nixos/lib/testing/network.nix` module. + + A few special options are available, that aren't in a plain NixOS configuration. See [Configuring the nodes](#sec-nixos-test-nodes) + ''; + }; + + defaults = mkOption { + description = mdDoc '' + NixOS configuration that is applied to all [{option}`nodes`](#test-opt-nodes). + ''; + type = types.deferredModule; + default = { }; + }; + + extraBaseModules = mkOption { + description = mdDoc '' + NixOS configuration that, like [{option}`defaults`](#test-opt-defaults), is applied to all [{option}`nodes`](#test-opt-nodes) and can not be undone with [`specialisation.<name>.inheritParentConfig`](https://search.nixos.org/options?show=specialisation.%3Cname%3E.inheritParentConfig&from=0&size=50&sort=relevance&type=packages&query=specialisation). + ''; + type = types.deferredModule; + default = { }; + }; + + node.pkgs = mkOption { + description = mdDoc '' + The Nixpkgs to use for the nodes. + + Setting this will make the `nixpkgs.*` options read-only, to avoid mistakenly testing with a Nixpkgs configuration that diverges from regular use. + ''; + type = types.nullOr types.pkgs; + default = null; + defaultText = literalMD '' + `null`, so construct `pkgs` according to the `nixpkgs.*` options as usual. + ''; + }; + + node.pkgsReadOnly = mkOption { + description = mdDoc '' + Whether to make the `nixpkgs.*` options read-only. This is only relevant when [`node.pkgs`](#test-opt-node.pkgs) is set. + + Set this to `false` when any of the [`nodes`](#test-opt-nodes) needs to configure any of the `nixpkgs.*` options. This will slow down evaluation of your test a bit. + ''; + type = types.bool; + default = config.node.pkgs != null; + defaultText = literalExpression ''node.pkgs != null''; + }; + + node.specialArgs = mkOption { + type = types.lazyAttrsOf types.raw; + default = { }; + description = mdDoc '' + An attribute set of arbitrary values that will be made available as module arguments during the resolution of module `imports`. + + Note that it is not possible to override these from within the NixOS configurations. If you argument is not relevant to `imports`, consider setting {option}`defaults._module.args.<name>` instead. + ''; + }; + + nodesCompat = mkOption { + internal = true; + description = mdDoc '' + Basically `_module.args.nodes`, but with backcompat and warnings added. + + This will go away. + ''; + }; + }; + + config = { + _module.args.nodes = config.nodesCompat; + nodesCompat = + mapAttrs + (name: config: config // { + config = lib.warnIf (lib.isInOldestRelease 2211) + "Module argument `nodes.${name}.config` is deprecated. Use `nodes.${name}` instead." + config; + }) + config.nodes; + + passthru.nodes = config.nodesCompat; + + defaults = mkIf config.node.pkgsReadOnly { + nixpkgs.pkgs = config.node.pkgs; + imports = [ ../../modules/misc/nixpkgs/read-only.nix ]; + }; + + }; +} diff --git a/nixpkgs/nixos/lib/testing/pkgs.nix b/nixpkgs/nixos/lib/testing/pkgs.nix new file mode 100644 index 000000000000..22dd586868e3 --- /dev/null +++ b/nixpkgs/nixos/lib/testing/pkgs.nix @@ -0,0 +1,11 @@ +{ config, lib, hostPkgs, ... }: +{ + config = { + # default pkgs for use in VMs + _module.args.pkgs = hostPkgs; + + defaults = { + # TODO: a module to set a shared pkgs, if options.nixpkgs.* is untouched by user (highestPrio) */ + }; + }; +} diff --git a/nixpkgs/nixos/lib/testing/run.nix b/nixpkgs/nixos/lib/testing/run.nix new file mode 100644 index 000000000000..9440c1acdfd8 --- /dev/null +++ b/nixpkgs/nixos/lib/testing/run.nix @@ -0,0 +1,67 @@ +{ config, hostPkgs, lib, ... }: +let + inherit (lib) types mkOption mdDoc; +in +{ + options = { + passthru = mkOption { + type = types.lazyAttrsOf types.raw; + description = mdDoc '' + Attributes to add to the returned derivations, + which are not necessarily part of the build. + + This is a bit like doing `drv // { myAttr = true; }` (which would be lost by `overrideAttrs`). + It does not change the actual derivation, but adds the attribute nonetheless, so that + consumers of what would be `drv` have more information. + ''; + }; + + rawTestDerivation = mkOption { + type = types.package; + description = mdDoc '' + Unfiltered version of `test`, for troubleshooting the test framework and `testBuildFailure` in the test framework's test suite. + This is not intended for general use. Use `test` instead. + ''; + internal = true; + }; + + test = mkOption { + type = types.package; + # TODO: can the interactive driver be configured to access the network? + description = mdDoc '' + Derivation that runs the test as its "build" process. + + This implies that NixOS tests run isolated from the network, making them + more dependable. + ''; + }; + }; + + config = { + rawTestDerivation = hostPkgs.stdenv.mkDerivation { + name = "vm-test-run-${config.name}"; + + requiredSystemFeatures = [ "kvm" "nixos-test" ]; + + buildCommand = '' + mkdir -p $out + + # effectively mute the XMLLogger + export LOGFILE=/dev/null + + ${config.driver}/bin/nixos-test-driver -o $out + ''; + + passthru = config.passthru; + + meta = config.meta; + }; + test = lib.lazyDerivation { # lazyDerivation improves performance when only passthru items and/or meta are used. + derivation = config.rawTestDerivation; + inherit (config) passthru meta; + }; + + # useful for inspection (debugging / exploration) + passthru.config = config; + }; +} diff --git a/nixpkgs/nixos/lib/testing/testScript.nix b/nixpkgs/nixos/lib/testing/testScript.nix new file mode 100644 index 000000000000..5c36d754d79d --- /dev/null +++ b/nixpkgs/nixos/lib/testing/testScript.nix @@ -0,0 +1,84 @@ +testModuleArgs@{ config, lib, hostPkgs, nodes, moduleType, ... }: +let + inherit (lib) mkOption types mdDoc; + inherit (types) either str functionTo; +in +{ + options = { + testScript = mkOption { + type = either str (functionTo str); + description = mdDoc '' + A series of python declarations and statements that you write to perform + the test. + ''; + }; + testScriptString = mkOption { + type = str; + readOnly = true; + internal = true; + }; + + includeTestScriptReferences = mkOption { + type = types.bool; + default = true; + internal = true; + }; + withoutTestScriptReferences = mkOption { + type = moduleType; + description = mdDoc '' + A parallel universe where the testScript is invalid and has no references. + ''; + internal = true; + visible = false; + }; + }; + config = { + withoutTestScriptReferences.includeTestScriptReferences = false; + withoutTestScriptReferences.testScript = lib.mkForce "testscript omitted"; + + testScriptString = + if lib.isFunction config.testScript + then + config.testScript + { + nodes = + lib.mapAttrs + (k: v: + if v.virtualisation.useNixStoreImage + then + # prevent infinite recursion when testScript would + # reference v's toplevel + config.withoutTestScriptReferences.nodesCompat.${k} + else + # reuse memoized config + v + ) + config.nodesCompat; + } + else config.testScript; + + defaults = { config, name, ... }: { + # Make sure all derivations referenced by the test + # script are available on the nodes. When the store is + # accessed through 9p, this isn't important, since + # everything in the store is available to the guest, + # but when building a root image it is, as all paths + # that should be available to the guest has to be + # copied to the image. + virtualisation.additionalPaths = + lib.optional + # A testScript may evaluate nodes, which has caused + # infinite recursions. The demand cycle involves: + # testScript --> + # nodes --> + # toplevel --> + # additionalPaths --> + # hasContext testScript' --> + # testScript (ad infinitum) + # If we don't need to build an image, we can break this + # cycle by short-circuiting when useNixStoreImage is false. + (config.virtualisation.useNixStoreImage && builtins.hasContext testModuleArgs.config.testScriptString && testModuleArgs.config.includeTestScriptReferences) + (hostPkgs.writeStringReferencesToFile testModuleArgs.config.testScriptString); + }; + }; +} diff --git a/nixpkgs/nixos/lib/utils.nix b/nixpkgs/nixos/lib/utils.nix new file mode 100644 index 000000000000..e618cf2f861a --- /dev/null +++ b/nixpkgs/nixos/lib/utils.nix @@ -0,0 +1,236 @@ +{ lib, config, pkgs }: with lib; + +rec { + + # Copy configuration files to avoid having the entire sources in the system closure + copyFile = filePath: pkgs.runCommand (builtins.unsafeDiscardStringContext (builtins.baseNameOf filePath)) {} '' + cp ${filePath} $out + ''; + + # Check whenever fileSystem is needed for boot. NOTE: Make sure + # pathsNeededForBoot is closed under the parent relationship, i.e. if /a/b/c + # is in the list, put /a and /a/b in as well. + pathsNeededForBoot = [ "/" "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/var/lib/nixos" "/etc" "/usr" ]; + fsNeededForBoot = fs: fs.neededForBoot || elem fs.mountPoint pathsNeededForBoot; + + # Check whenever `b` depends on `a` as a fileSystem + fsBefore = a: b: + let + # normalisePath adds a slash at the end of the path if it didn't already + # have one. + # + # The reason slashes are added at the end of each path is to prevent `b` + # from accidentally depending on `a` in cases like + # a = { mountPoint = "/aaa"; ... } + # b = { device = "/aaaa"; ... } + # Here a.mountPoint *is* a prefix of b.device even though a.mountPoint is + # *not* a parent of b.device. If we add a slash at the end of each string, + # though, this is not a problem: "/aaa/" is not a prefix of "/aaaa/". + normalisePath = path: "${path}${optionalString (!(hasSuffix "/" path)) "/"}"; + normalise = mount: mount // { device = normalisePath (toString mount.device); + mountPoint = normalisePath mount.mountPoint; + depends = map normalisePath mount.depends; + }; + + a' = normalise a; + b' = normalise b; + + in hasPrefix a'.mountPoint b'.device + || hasPrefix a'.mountPoint b'.mountPoint + || any (hasPrefix a'.mountPoint) b'.depends; + + # Escape a path according to the systemd rules. FIXME: slow + # The rules are described in systemd.unit(5) as follows: + # The escaping algorithm operates as follows: given a string, any "/" character is replaced by "-", and all other characters which are not ASCII alphanumerics, ":", "_" or "." are replaced by C-style "\x2d" escapes. In addition, "." is replaced with such a C-style escape when it would appear as the first character in the escaped string. + # When the input qualifies as absolute file system path, this algorithm is extended slightly: the path to the root directory "/" is encoded as single dash "-". In addition, any leading, trailing or duplicate "/" characters are removed from the string before transformation. Example: /foo//bar/baz/ becomes "foo-bar-baz". + escapeSystemdPath = s: let + replacePrefix = p: r: s: (if (hasPrefix p s) then r + (removePrefix p s) else s); + trim = s: removeSuffix "/" (removePrefix "/" s); + normalizedPath = strings.normalizePath s; + in + replaceStrings ["/"] ["-"] + (replacePrefix "." (strings.escapeC ["."] ".") + (strings.escapeC (stringToCharacters " !\"#$%&'()*+,;<=>=@[\\]^`{|}~-") + (if normalizedPath == "/" then normalizedPath else trim normalizedPath))); + + # Quotes an argument for use in Exec* service lines. + # systemd accepts "-quoted strings with escape sequences, toJSON produces + # a subset of these. + # Additionally we escape % to disallow expansion of % specifiers. Any lone ; + # in the input will be turned it ";" and thus lose its special meaning. + # Every $ is escaped to $$, this makes it unnecessary to disable environment + # substitution for the directive. + escapeSystemdExecArg = arg: + let + s = if builtins.isPath arg then "${arg}" + else if builtins.isString arg then arg + else if builtins.isInt arg || builtins.isFloat arg then toString arg + else throw "escapeSystemdExecArg only allows strings, paths and numbers"; + in + replaceStrings [ "%" "$" ] [ "%%" "$$" ] (builtins.toJSON s); + + # Quotes a list of arguments into a single string for use in a Exec* + # line. + escapeSystemdExecArgs = concatMapStringsSep " " escapeSystemdExecArg; + + # Returns a system path for a given shell package + toShellPath = shell: + if types.shellPackage.check shell then + "/run/current-system/sw${shell.shellPath}" + else if types.package.check shell then + throw "${shell} is not a shell package" + else + shell; + + /* Recurse into a list or an attrset, searching for attrs named like + the value of the "attr" parameter, and return an attrset where the + names are the corresponding jq path where the attrs were found and + the values are the values of the attrs. + + Example: + recursiveGetAttrWithJqPrefix { + example = [ + { + irrelevant = "not interesting"; + } + { + ignored = "ignored attr"; + relevant = { + secret = { + _secret = "/path/to/secret"; + }; + }; + } + ]; + } "_secret" -> { ".example[1].relevant.secret" = "/path/to/secret"; } + */ + recursiveGetAttrWithJqPrefix = item: attr: + let + recurse = prefix: item: + if item ? ${attr} then + nameValuePair prefix item.${attr} + else if isAttrs item then + map (name: + let + escapedName = ''"${replaceStrings [''"'' "\\"] [''\"'' "\\\\"] name}"''; + in + recurse (prefix + "." + escapedName) item.${name}) (attrNames item) + else if isList item then + imap0 (index: item: recurse (prefix + "[${toString index}]") item) item + else + []; + in listToAttrs (flatten (recurse "" item)); + + /* Takes an attrset and a file path and generates a bash snippet that + outputs a JSON file at the file path with all instances of + + { _secret = "/path/to/secret" } + + in the attrset replaced with the contents of the file + "/path/to/secret" in the output JSON. + + When a configuration option accepts an attrset that is finally + converted to JSON, this makes it possible to let the user define + arbitrary secret values. + + Example: + If the file "/path/to/secret" contains the string + "topsecretpassword1234", + + genJqSecretsReplacementSnippet { + example = [ + { + irrelevant = "not interesting"; + } + { + ignored = "ignored attr"; + relevant = { + secret = { + _secret = "/path/to/secret"; + }; + }; + } + ]; + } "/path/to/output.json" + + would generate a snippet that, when run, outputs the following + JSON file at "/path/to/output.json": + + { + "example": [ + { + "irrelevant": "not interesting" + }, + { + "ignored": "ignored attr", + "relevant": { + "secret": "topsecretpassword1234" + } + } + ] + } + */ + genJqSecretsReplacementSnippet = genJqSecretsReplacementSnippet' "_secret"; + + # Like genJqSecretsReplacementSnippet, but allows the name of the + # attr which identifies the secret to be changed. + genJqSecretsReplacementSnippet' = attr: set: output: + let + secrets = recursiveGetAttrWithJqPrefix set attr; + stringOrDefault = str: def: if str == "" then def else str; + in '' + if [[ -h '${output}' ]]; then + rm '${output}' + fi + + inherit_errexit_enabled=0 + shopt -pq inherit_errexit && inherit_errexit_enabled=1 + shopt -s inherit_errexit + '' + + concatStringsSep + "\n" + (imap1 (index: name: '' + secret${toString index}=$(<'${secrets.${name}}') + export secret${toString index} + '') + (attrNames secrets)) + + "\n" + + "${pkgs.jq}/bin/jq >'${output}' " + + lib.escapeShellArg (stringOrDefault + (concatStringsSep + " | " + (imap1 (index: name: ''${name} = $ENV.secret${toString index}'') + (attrNames secrets))) + ".") + + '' + <<'EOF' + ${builtins.toJSON set} + EOF + (( ! $inherit_errexit_enabled )) && shopt -u inherit_errexit + ''; + + /* Remove packages of packagesToRemove from packages, based on their names. + Relies on package names and has quadratic complexity so use with caution! + + Type: + removePackagesByName :: [package] -> [package] -> [package] + + Example: + removePackagesByName [ nautilus file-roller ] [ file-roller totem ] + => [ nautilus ] + */ + removePackagesByName = packages: packagesToRemove: + let + namesToRemove = map lib.getName packagesToRemove; + in + lib.filter (x: !(builtins.elem (lib.getName x) namesToRemove)) packages; + + systemdUtils = { + lib = import ./systemd-lib.nix { inherit lib config pkgs; }; + unitOptions = import ./systemd-unit-options.nix { inherit lib systemdUtils; }; + types = import ./systemd-types.nix { inherit lib systemdUtils pkgs; }; + network = { + units = import ./systemd-network-units.nix { inherit lib systemdUtils; }; + }; + }; +} |