diff options
Diffstat (limited to 'nixpkgs/nixos/maintainers')
8 files changed, 684 insertions, 0 deletions
diff --git a/nixpkgs/nixos/maintainers/option-usages.nix b/nixpkgs/nixos/maintainers/option-usages.nix new file mode 100644 index 000000000000..11247666ecda --- /dev/null +++ b/nixpkgs/nixos/maintainers/option-usages.nix @@ -0,0 +1,192 @@ +{ configuration ? import ../lib/from-env.nix "NIXOS_CONFIG" <nixos-config> + +# provide an option name, as a string literal. +, testOption ? null + +# provide a list of option names, as string literals. +, testOptions ? [ ] +}: + +# This file is made to be used as follow: +# +# $ nix-instantiate ./option-usage.nix --argstr testOption service.xserver.enable -A txtContent --eval +# +# or +# +# $ nix-build ./option-usage.nix --argstr testOption service.xserver.enable -A txt -o service.xserver.enable._txt +# +# Other targets exists such as `dotContent`, `dot`, and `pdf`. If you are +# looking for the option usage of multiple options, you can provide a list +# as argument. +# +# $ nix-build ./option-usage.nix --arg testOptions \ +# '["boot.loader.gummiboot.enable" "boot.loader.gummiboot.timeout"]' \ +# -A txt -o gummiboot.list +# +# Note, this script is slow as it has to evaluate all options of the system +# once per queried option. +# +# This nix expression works by doing a first evaluation, which evaluates the +# result of every option. +# +# Then, for each queried option, we evaluate the NixOS modules a second +# time, except that we replace the `config` argument of all the modules with +# the result of the original evaluation, except for the tested option which +# value is replaced by a `throw` statement which is caught by the `tryEval` +# evaluation of each option value. +# +# We then compare the result of the evaluation of the original module, with +# the result of the second evaluation, and consider that the new failures are +# caused by our mutation of the `config` argument. +# +# Doing so returns all option results which are directly using the +# tested option result. + +with import ../../lib; + +let + + evalFun = { + specialArgs ? {} + }: import ../lib/eval-config.nix { + modules = [ configuration ]; + inherit specialArgs; + }; + + eval = evalFun {}; + inherit (eval) pkgs; + + excludedTestOptions = [ + # We cannot evluate _module.args, as it is used during the computation + # of the modules list. + "_module.args" + + # For some reasons which we yet have to investigate, some options cannot + # be replaced by a throw without causing a non-catchable failure. + "networking.bonds" + "networking.bridges" + "networking.interfaces" + "networking.macvlans" + "networking.sits" + "networking.vlans" + "services.openssh.startWhenNeeded" + ]; + + # for some reasons which we yet have to investigate, some options are + # time-consuming to compute, thus we filter them out at the moment. + excludedOptions = [ + "boot.systemd.services" + "systemd.services" + "kde.extraPackages" + ]; + excludeOptions = list: + filter (opt: !(elem (showOption opt.loc) excludedOptions)) list; + + + reportNewFailures = old: new: + let + filterChanges = + filter ({fst, snd}: + !(fst.success -> snd.success) + ); + + keepNames = + map ({fst, snd}: + /* assert fst.name == snd.name; */ snd.name + ); + + # Use tryEval (strict ...) to know if there is any failure while + # evaluating the option value. + # + # Note, the `strict` function is not strict enough, but using toXML + # builtins multiply by 4 the memory usage and the time used to compute + # each options. + tryCollectOptions = moduleResult: + forEach (excludeOptions (collect isOption moduleResult)) (opt: + { name = showOption opt.loc; } // builtins.tryEval (strict opt.value)); + in + keepNames ( + filterChanges ( + zipLists (tryCollectOptions old) (tryCollectOptions new) + ) + ); + + + # Create a list of modules where each module contains only one failling + # options. + introspectionModules = + let + setIntrospection = opt: rec { + name = showOption opt.loc; + path = opt.loc; + config = setAttrByPath path + (throw "Usage introspection of '${name}' by forced failure."); + }; + in + map setIntrospection (collect isOption eval.options); + + overrideConfig = thrower: + recursiveUpdateUntil (path: old: new: + path == thrower.path + ) eval.config thrower.config; + + + graph = + map (thrower: { + option = thrower.name; + usedBy = assert __trace "Investigate ${thrower.name}" true; + reportNewFailures eval.options (evalFun { + specialArgs = { + config = overrideConfig thrower; + }; + }).options; + }) introspectionModules; + + displayOptionsGraph = + let + checkList = + if testOption != null then [ testOption ] + else testOptions; + checkAll = checkList == []; + in + flip filter graph ({option, ...}: + (checkAll || elem option checkList) + && !(elem option excludedTestOptions) + ); + + graphToDot = graph: '' + digraph "Option Usages" { + ${concatMapStrings ({option, usedBy}: + concatMapStrings (user: '' + "${option}" -> "${user}"'' + ) usedBy + ) displayOptionsGraph} + } + ''; + + graphToText = graph: + concatMapStrings ({usedBy, ...}: + concatMapStrings (user: '' + ${user} + '') usedBy + ) displayOptionsGraph; + +in + +rec { + dotContent = graphToDot graph; + dot = pkgs.writeTextFile { + name = "option_usages.dot"; + text = dotContent; + }; + + pdf = pkgs.texFunctions.dot2pdf { + dotGraph = dot; + }; + + txtContent = graphToText graph; + txt = pkgs.writeTextFile { + name = "option_usages.txt"; + text = txtContent; + }; +} diff --git a/nixpkgs/nixos/maintainers/scripts/azure/create-azure.sh b/nixpkgs/nixos/maintainers/scripts/azure/create-azure.sh new file mode 100755 index 000000000000..0558f8dfffcb --- /dev/null +++ b/nixpkgs/nixos/maintainers/scripts/azure/create-azure.sh @@ -0,0 +1,8 @@ +#! /bin/sh -eu + +export NIX_PATH=nixpkgs=$(dirname $(readlink -f $0))/../../../.. +export NIXOS_CONFIG=$(dirname $(readlink -f $0))/../../../modules/virtualisation/azure-image.nix +export TIMESTAMP=$(date +%Y%m%d%H%M) + +nix-build '<nixpkgs/nixos>' \ + -A config.system.build.azureImage --argstr system x86_64-linux -o azure -j 10 diff --git a/nixpkgs/nixos/maintainers/scripts/azure/upload-azure.sh b/nixpkgs/nixos/maintainers/scripts/azure/upload-azure.sh new file mode 100755 index 000000000000..2ea35d1d4c33 --- /dev/null +++ b/nixpkgs/nixos/maintainers/scripts/azure/upload-azure.sh @@ -0,0 +1,22 @@ +#! /bin/sh -e + +export STORAGE=${STORAGE:-nixos} +export THREADS=${THREADS:-8} + +azure-vhd-utils-for-go upload --localvhdpath azure/disk.vhd --stgaccountname "$STORAGE" --stgaccountkey "$KEY" \ + --containername images --blobname nixos-unstable-nixops-updated.vhd --parallelism "$THREADS" --overwrite + + + + + + + + + + + + + + + diff --git a/nixpkgs/nixos/maintainers/scripts/cloudstack/cloudstack-image.nix b/nixpkgs/nixos/maintainers/scripts/cloudstack/cloudstack-image.nix new file mode 100644 index 000000000000..37b46db059c0 --- /dev/null +++ b/nixpkgs/nixos/maintainers/scripts/cloudstack/cloudstack-image.nix @@ -0,0 +1,23 @@ +# nix-build '<nixpkgs/nixos>' -A config.system.build.cloudstackImage --arg configuration "{ imports = [ ./nixos/maintainers/scripts/cloudstack/cloudstack-image.nix ]; }" + +{ config, lib, pkgs, ... }: + +with lib; + +{ + imports = + [ ../../../modules/virtualisation/cloudstack-config.nix ]; + + system.build.cloudstackImage = import ../../../lib/make-disk-image.nix { + inherit lib config pkgs; + diskSize = 8192; + format = "qcow2"; + configFile = pkgs.writeText "configuration.nix" + '' + { + imports = [ <nixpkgs/nixos/modules/virtualisation/cloudstack-config.nix> ]; + } + ''; + }; + +} diff --git a/nixpkgs/nixos/maintainers/scripts/ec2/amazon-image.nix b/nixpkgs/nixos/maintainers/scripts/ec2/amazon-image.nix new file mode 100644 index 000000000000..36f3e7af873d --- /dev/null +++ b/nixpkgs/nixos/maintainers/scripts/ec2/amazon-image.nix @@ -0,0 +1,94 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.amazonImage; +in { + + imports = [ ../../../modules/virtualisation/amazon-image.nix ]; + + # Amazon recomments setting this to the highest possible value for a good EBS + # experience, which prior to 4.15 was 255. + # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nvme-ebs-volumes.html#timeout-nvme-ebs-volumes + config.boot.kernelParams = + let timeout = + if pkgs.lib.versionAtLeast config.boot.kernelPackages.kernel.version "4.15" + then "4294967295" + else "255"; + in [ "nvme_core.io_timeout=${timeout}" ]; + + options.amazonImage = { + name = mkOption { + type = types.str; + description = "The name of the generated derivation"; + default = "nixos-amazon-image-${config.system.nixos.label}-${pkgs.stdenv.hostPlatform.system}"; + }; + + contents = mkOption { + example = literalExample '' + [ { source = pkgs.memtest86 + "/memtest.bin"; + target = "boot/memtest.bin"; + } + ] + ''; + default = []; + description = '' + This option lists files to be copied to fixed locations in the + generated image. Glob patterns work. + ''; + }; + + sizeMB = mkOption { + type = types.int; + default = if config.ec2.hvm then 2048 else 8192; + description = "The size in MB of the image"; + }; + + format = mkOption { + type = types.enum [ "raw" "qcow2" "vpc" ]; + default = "vpc"; + description = "The image format to output"; + }; + }; + + config.system.build.amazonImage = import ../../../lib/make-disk-image.nix { + inherit lib config; + inherit (cfg) contents format name; + pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package + partitionTableType = if config.ec2.efi then "efi" + else if config.ec2.hvm then "legacy" + else "none"; + diskSize = cfg.sizeMB; + fsType = "ext4"; + configFile = pkgs.writeText "configuration.nix" + '' + { + imports = [ <nixpkgs/nixos/modules/virtualisation/amazon-image.nix> ]; + ${optionalString config.ec2.hvm '' + ec2.hvm = true; + ''} + ${optionalString config.ec2.efi '' + ec2.efi = true; + ''} + } + ''; + postVM = '' + extension=''${diskImage##*.} + friendlyName=$out/${cfg.name}.$extension + mv "$diskImage" "$friendlyName" + diskImage=$friendlyName + + mkdir -p $out/nix-support + echo "file ${cfg.format} $diskImage" >> $out/nix-support/hydra-build-products + + ${pkgs.jq}/bin/jq -n \ + --arg label ${lib.escapeShellArg config.system.nixos.label} \ + --arg system ${lib.escapeShellArg pkgs.stdenv.hostPlatform.system} \ + --arg logical_bytes "$(${pkgs.qemu}/bin/qemu-img info --output json "$diskImage" | ${pkgs.jq}/bin/jq '."virtual-size"')" \ + --arg file "$diskImage" \ + '$ARGS.named' \ + > $out/nix-support/image-info.json + ''; + }; +} diff --git a/nixpkgs/nixos/maintainers/scripts/ec2/create-amis.sh b/nixpkgs/nixos/maintainers/scripts/ec2/create-amis.sh new file mode 100755 index 000000000000..145eb49ced7a --- /dev/null +++ b/nixpkgs/nixos/maintainers/scripts/ec2/create-amis.sh @@ -0,0 +1,296 @@ +#!/usr/bin/env nix-shell +#!nix-shell -p awscli -p jq -p qemu -i bash + +# Uploads and registers NixOS images built from the +# <nixos/release.nix> amazonImage attribute. Images are uploaded and +# registered via a home region, and then copied to other regions. + +# The home region requires an s3 bucket, and a "vmimport" IAM role +# with access to the S3 bucket. Configuration of the vmimport role is +# documented in +# https://docs.aws.amazon.com/vm-import/latest/userguide/vmimport-image-import.html + +# set -x +set -euo pipefail + +# configuration +state_dir=$HOME/amis/ec2-images +home_region=eu-west-1 +bucket=nixos-amis + +regions=(eu-west-1 eu-west-2 eu-west-3 eu-central-1 eu-north-1 + us-east-1 us-east-2 us-west-1 us-west-2 + ca-central-1 + ap-southeast-1 ap-southeast-2 ap-northeast-1 ap-northeast-2 + ap-south-1 ap-east-1 + sa-east-1) + +log() { + echo "$@" >&2 +} + +if [ -z "$1" ]; then + log "Usage: ./upload-amazon-image.sh IMAGE_OUTPUT" + exit 1 +fi + +# result of the amazon-image from nixos/release.nix +store_path=$1 + +if [ ! -e "$store_path" ]; then + log "Store path: $store_path does not exist, fetching..." + nix-store --realise "$store_path" +fi + +if [ ! -d "$store_path" ]; then + log "store_path: $store_path is not a directory. aborting" + exit 1 +fi + +read_image_info() { + if [ ! -e "$store_path/nix-support/image-info.json" ]; then + log "Image missing metadata" + exit 1 + fi + jq -r "$1" "$store_path/nix-support/image-info.json" +} + +# We handle a single image per invocation, store all attributes in +# globals for convenience. +image_label=$(read_image_info .label) +image_system=$(read_image_info .system) +image_file=$(read_image_info .file) +image_logical_bytes=$(read_image_info .logical_bytes) + +# Derived attributes + +image_logical_gigabytes=$((($image_logical_bytes-1)/1024/1024/1024+1)) # Round to the next GB + +case "$image_system" in + aarch64-linux) + amazon_arch=arm64 + ;; + x86_64-linux) + amazon_arch=x86_64 + ;; + *) + log "Unknown system: $image_system" + exit 1 +esac + +image_name="NixOS-${image_label}-${image_system}" +image_description="NixOS ${image_label} ${image_system}" + +log "Image Details:" +log " Name: $image_name" +log " Description: $image_description" +log " Size (gigabytes): $image_logical_gigabytes" +log " System: $image_system" +log " Amazon Arch: $amazon_arch" + +read_state() { + local state_key=$1 + local type=$2 + + cat "$state_dir/$state_key.$type" 2>/dev/null || true +} + +write_state() { + local state_key=$1 + local type=$2 + local val=$3 + + mkdir -p $state_dir + echo "$val" > "$state_dir/$state_key.$type" +} + +wait_for_import() { + local region=$1 + local task_id=$2 + local state snapshot_id + log "Waiting for import task $task_id to be completed" + while true; do + read state progress snapshot_id < <( + aws ec2 describe-import-snapshot-tasks --region $region --import-task-ids "$task_id" | \ + jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail | "\(.Status) \(.Progress) \(.SnapshotId)"' + ) + log " ... state=$state progress=$progress snapshot_id=$snapshot_id" + case "$state" in + active) + sleep 10 + ;; + completed) + echo "$snapshot_id" + return + ;; + *) + log "Unexpected snapshot import state: '${state}'" + exit 1 + ;; + esac + done +} + +wait_for_image() { + local region=$1 + local ami_id=$2 + local state + log "Waiting for image $ami_id to be available" + + while true; do + read state < <( + aws ec2 describe-images --image-ids "$ami_id" --region $region | \ + jq -r ".Images[].State" + ) + log " ... state=$state" + case "$state" in + pending) + sleep 10 + ;; + available) + return + ;; + *) + log "Unexpected AMI state: '${state}'" + exit 1 + ;; + esac + done +} + + +make_image_public() { + local region=$1 + local ami_id=$2 + + wait_for_image $region "$ami_id" + + log "Making image $ami_id public" + + aws ec2 modify-image-attribute \ + --image-id "$ami_id" --region "$region" --launch-permission 'Add={Group=all}' >&2 +} + +upload_image() { + local region=$1 + + local aws_path=${image_file#/} + + local state_key="$region.$image_label.$image_system" + local task_id=$(read_state "$state_key" task_id) + local snapshot_id=$(read_state "$state_key" snapshot_id) + local ami_id=$(read_state "$state_key" ami_id) + + if [ -z "$task_id" ]; then + log "Checking for image on S3" + if ! aws s3 ls --region "$region" "s3://${bucket}/${aws_path}" >&2; then + log "Image missing from aws, uploading" + aws s3 cp --region $region "$image_file" "s3://${bucket}/${aws_path}" >&2 + fi + + log "Importing image from S3 path s3://$bucket/$aws_path" + + task_id=$(aws ec2 import-snapshot --disk-container "{ + \"Description\": \"nixos-image-${image_label}-${image_system}\", + \"Format\": \"vhd\", + \"UserBucket\": { + \"S3Bucket\": \"$bucket\", + \"S3Key\": \"$aws_path\" + } + }" --region $region | jq -r '.ImportTaskId') + + write_state "$state_key" task_id "$task_id" + fi + + if [ -z "$snapshot_id" ]; then + snapshot_id=$(wait_for_import "$region" "$task_id") + write_state "$state_key" snapshot_id "$snapshot_id" + fi + + if [ -z "$ami_id" ]; then + log "Registering snapshot $snapshot_id as AMI" + + local block_device_mappings=( + "DeviceName=/dev/xvda,Ebs={SnapshotId=$snapshot_id,VolumeSize=$image_logical_gigabytes,DeleteOnTermination=true,VolumeType=gp2}" + ) + + local extra_flags=( + --root-device-name /dev/xvda + --sriov-net-support simple + --ena-support + --virtualization-type hvm + ) + + block_device_mappings+=(DeviceName=/dev/sdb,VirtualName=ephemeral0) + block_device_mappings+=(DeviceName=/dev/sdc,VirtualName=ephemeral1) + block_device_mappings+=(DeviceName=/dev/sdd,VirtualName=ephemeral2) + block_device_mappings+=(DeviceName=/dev/sde,VirtualName=ephemeral3) + + ami_id=$( + aws ec2 register-image \ + --name "$image_name" \ + --description "$image_description" \ + --region $region \ + --architecture $amazon_arch \ + --block-device-mappings "${block_device_mappings[@]}" \ + "${extra_flags[@]}" \ + | jq -r '.ImageId' + ) + + write_state "$state_key" ami_id "$ami_id" + fi + + make_image_public $region "$ami_id" + + echo "$ami_id" +} + +copy_to_region() { + local region=$1 + local from_region=$2 + local from_ami_id=$3 + + state_key="$region.$image_label.$image_system" + ami_id=$(read_state "$state_key" ami_id) + + if [ -z "$ami_id" ]; then + log "Copying $from_ami_id to $region" + ami_id=$( + aws ec2 copy-image \ + --region "$region" \ + --source-region "$from_region" \ + --source-image-id "$from_ami_id" \ + --name "$image_name" \ + --description "$image_description" \ + | jq -r '.ImageId' + ) + + write_state "$state_key" ami_id "$ami_id" + fi + + make_image_public $region "$ami_id" + + echo "$ami_id" +} + +upload_all() { + home_image_id=$(upload_image "$home_region") + jq -n \ + --arg key "$home_region.$image_system" \ + --arg value "$home_image_id" \ + '$ARGS.named' + + for region in "${regions[@]}"; do + if [ "$region" = "$home_region" ]; then + continue + fi + copied_image_id=$(copy_to_region "$region" "$home_region" "$home_image_id") + + jq -n \ + --arg key "$region.$image_system" \ + --arg value "$copied_image_id" \ + '$ARGS.named' + done +} + +upload_all | jq --slurp from_entries diff --git a/nixpkgs/nixos/maintainers/scripts/gce/create-gce.sh b/nixpkgs/nixos/maintainers/scripts/gce/create-gce.sh new file mode 100755 index 000000000000..77cc64e591e9 --- /dev/null +++ b/nixpkgs/nixos/maintainers/scripts/gce/create-gce.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env nix-shell +#! nix-shell -i bash -p google-cloud-sdk + +set -euo pipefail + +BUCKET_NAME="${BUCKET_NAME:-nixos-cloud-images}" +TIMESTAMP="$(date +%Y%m%d%H%M)" +export TIMESTAMP + +nix-build '<nixpkgs/nixos/lib/eval-config.nix>' \ + -A config.system.build.googleComputeImage \ + --arg modules "[ <nixpkgs/nixos/modules/virtualisation/google-compute-image.nix> ]" \ + --argstr system x86_64-linux \ + -o gce \ + -j 10 + +img_path=$(echo gce/*.tar.gz) +img_name=${IMAGE_NAME:-$(basename "$img_path")} +img_id=$(echo "$img_name" | sed 's|.raw.tar.gz$||;s|\.|-|g;s|_|-|g') +if ! gsutil ls "gs://${BUCKET_NAME}/$img_name"; then + gsutil cp "$img_path" "gs://${BUCKET_NAME}/$img_name" + gsutil acl ch -u AllUsers:R "gs://${BUCKET_NAME}/$img_name" +fi diff --git a/nixpkgs/nixos/maintainers/scripts/openstack/openstack-image.nix b/nixpkgs/nixos/maintainers/scripts/openstack/openstack-image.nix new file mode 100644 index 000000000000..4c464f43f61d --- /dev/null +++ b/nixpkgs/nixos/maintainers/scripts/openstack/openstack-image.nix @@ -0,0 +1,26 @@ +# nix-build '<nixpkgs/nixos>' -A config.system.build.openstackImage --arg configuration "{ imports = [ ./nixos/maintainers/scripts/openstack/openstack-image.nix ]; }" + +{ config, lib, pkgs, ... }: + +with lib; + +{ + imports = + [ ../../../modules/installer/cd-dvd/channel.nix + ../../../modules/virtualisation/openstack-config.nix + ]; + + system.build.openstackImage = import ../../../lib/make-disk-image.nix { + inherit lib config; + pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package + diskSize = 8192; + format = "qcow2"; + configFile = pkgs.writeText "configuration.nix" + '' + { + imports = [ <nixpkgs/nixos/modules/virtualisation/openstack-config.nix> ]; + } + ''; + }; + +} |