Merge #4678: nixos iso-image: support USB booting

by using syslinux (i.e. support the dd-method in addition to unetbootin).
@vcunat tidied the PR by squashing closely related changes together.

7 files changed, 226 insertions, 87 deletions

diff --git a/nixos/lib/make-iso9660-image.nix b/nixos/lib/make-iso9660-image.nix
index 5ad546e9534d..b2409c6006bc 100644
--- a/nixos/lib/make-iso9660-image.nix
+++ b/nixos/lib/make-iso9660-image.nix
@@ -1,4 +1,4 @@
-{ stdenv, perl, cdrkit, pathsFromGraph
+{ stdenv, perl, pathsFromGraph, xorriso, syslinux
 
 , # The file name of the resulting ISO image.
   isoName ? "cd.iso"
@@ -22,12 +22,18 @@
 , # Whether this should be an efi-bootable El-Torito CD.
   efiBootable ? false
 
+, # Wheter this should be an hybrid CD (bootable from USB as well as CD).
+  usbBootable ? false
+
 , # The path (in the ISO file system) of the boot image.
   bootImage ? ""
 
 , # The path (in the ISO file system) of the efi boot image.
   efiBootImage ? ""
 
+, # The path (outside the ISO file system) of the isohybrid-mbr image.
+  isohybridMbrImage ? ""
+
 , # Whether to compress the resulting ISO image with bzip2.
   compressImage ? false
 
@@ -38,13 +44,14 @@
 
 assert bootable -> bootImage != "";
 assert efiBootable -> efiBootImage != "";
+assert usbBootable -> isohybridMbrImage != "";
 
 stdenv.mkDerivation {
   name = "iso9660-image";
   builder = ./make-iso9660-image.sh;
-  buildInputs = [perl cdrkit];
+  buildInputs = [perl xorriso syslinux];
 
-  inherit isoName bootable bootImage compressImage volumeID pathsFromGraph efiBootImage efiBootable;
+  inherit isoName bootable bootImage compressImage volumeID pathsFromGraph efiBootImage efiBootable isohybridMbrImage usbBootable;
 
   # !!! should use XML.
   sources = map (x: x.source) contents;
diff --git a/nixos/lib/make-iso9660-image.sh b/nixos/lib/make-iso9660-image.sh
index 675b5bb35148..c9a373794692 100644
--- a/nixos/lib/make-iso9660-image.sh
+++ b/nixos/lib/make-iso9660-image.sh
@@ -13,6 +13,20 @@ stripSlash() {
     if test "${res:0:1}" = /; then res=${res:1}; fi
 }
 
+# Escape potential equal signs (=) with backslash (\=)
+escapeEquals() {
+    echo "$1" | sed -e 's/\\/\\\\/g' -e 's/=/\\=/g'
+}
+
+# Queues an file/directory to be placed on the ISO.
+# An entry consists of a local source path (2) and
+# a destination path on the ISO (1).
+addPath() {
+    target="$1"
+    source="$2"
+    echo "$(escapeEquals "$target")=$(escapeEquals "$source")" >> pathlist
+}
+
 stripSlash "$bootImage"; bootImage="$res"
 
 
@@ -31,11 +45,20 @@ if test -n "$bootable"; then
         fi
     done
 
-    bootFlags="-b $bootImage -c .boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table"
+    isoBootFlags="-eltorito-boot ${bootImage}
+                  -eltorito-catalog .boot.cat
+                  -no-emul-boot -boot-load-size 4 -boot-info-table"
+fi
+
+if test -n "$usbBootable"; then
+    usbBootFlags="-isohybrid-mbr ${isohybridMbrImage}"
 fi
 
 if test -n "$efiBootable"; then
-    bootFlags="$bootFlags -eltorito-alt-boot -e $efiBootImage -no-emul-boot"
+    efiBootFlags="-eltorito-alt-boot
+                  -e $efiBootImage
+                  -no-emul-boot
+                  -isohybrid-gpt-basdat"
 fi
 
 touch pathlist
@@ -44,14 +67,14 @@ touch pathlist
 # Add the individual files.
 for ((i = 0; i < ${#targets_[@]}; i++)); do
     stripSlash "${targets_[$i]}"
-    echo "$res=${sources_[$i]}" >> pathlist
+    addPath "$res" "${sources_[$i]}"
 done
 
 
 # Add the closures of the top-level store objects.
 storePaths=$(perl $pathsFromGraph closure-*)
 for i in $storePaths; do
-    echo "${i:1}=$i" >> pathlist
+    addPath "${i:1}" "$i"
 done
 
 
@@ -59,7 +82,7 @@ done
 # nix-store --load-db.
 if [ -n "$object" ]; then
     printRegistration=1 perl $pathsFromGraph closure-* > nix-path-registration
-    echo "nix-path-registration=nix-path-registration" >> pathlist
+    addPath "nix-path-registration" "nix-path-registration"
 fi
 
 
@@ -70,22 +93,39 @@ for ((n = 0; n < ${#objects[*]}; n++)); do
     if test "$symlink" != "none"; then
         mkdir -p $(dirname ./$symlink)
         ln -s $object ./$symlink
-        echo "$symlink=./$symlink" >> pathlist
+        addPath "$symlink" "./$symlink"
     fi
 done
 
-# !!! what does this do?
-cat pathlist | sed -e 's/=\(.*\)=\(.*\)=/\\=\1=\2\\=/' | tee pathlist.safer
-
-
 mkdir -p $out/iso
-genCommand="genisoimage -iso-level 4 -r -J $bootFlags -hide-rr-moved -graft-points -path-list pathlist.safer ${volumeID:+-V $volumeID}"
-if test -z "$compressImage"; then
-    $genCommand -o $out/iso/$isoName
-else
-    $genCommand | bzip2 > $out/iso/$isoName.bz2
+
+xorriso="xorriso
+ -as mkisofs
+ -iso-level 3
+ -volid ${volumeID}
+ -appid nixos
+ -publisher nixos
+ -graft-points
+ -full-iso9660-filenames
+ ${isoBootFlags}
+ ${usbBootFlags}
+ ${efiBootFlags}
+ -r
+ -path-list pathlist
+ --sort-weight 0 /
+ --sort-weight 1 /isolinux" # Make sure isolinux is near the beginning of the ISO
+
+$xorriso -output $out/iso/$isoName
+
+if test -n "$usbBootable"; then
+    echo "Making image hybrid..."
+    isohybrid --uefi $out/iso/$isoName
 fi
 
+if test -n "$compressImage"; then
+    echo "Compressing image..."
+    bzip2 $out/iso/$isoName
+fi
 
 mkdir -p $out/nix-support
 echo $system > $out/nix-support/system
diff --git a/nixos/lib/test-driver/Machine.pm b/nixos/lib/test-driver/Machine.pm
index 85c2bfa88e1a..e0791692d3ef 100644
--- a/nixos/lib/test-driver/Machine.pm
+++ b/nixos/lib/test-driver/Machine.pm
@@ -37,6 +37,10 @@ sub new {
             if defined $args->{hda};
         $startCommand .= "-cdrom $args->{cdrom} "
             if defined $args->{cdrom};
+        $startCommand .= "-device piix3-usb-uhci -drive id=usbdisk,file=$args->{usb},if=none,readonly -device usb-storage,drive=usbdisk "
+            if defined $args->{usb};
+        $startCommand .= "-bios $args->{bios} "
+            if defined $args->{bios};
         $startCommand .= $args->{qemuFlags} || "";
     } else {
         $startCommand = Cwd::abs_path $startCommand;
diff --git a/nixos/modules/installer/cd-dvd/installation-cd-base.nix b/nixos/modules/installer/cd-dvd/installation-cd-base.nix
index b723a91e4f35..4896eee29084 100644
--- a/nixos/modules/installer/cd-dvd/installation-cd-base.nix
+++ b/nixos/modules/installer/cd-dvd/installation-cd-base.nix
@@ -36,6 +36,9 @@ with lib;
   # EFI booting
   isoImage.makeEfiBootable = true;
 
+  # USB booting
+  isoImage.makeUsbBootable = true;
+
   # Add Memtest86+ to the CD.
   boot.loader.grub.memtest86.enable = true;
 
diff --git a/nixos/modules/installer/cd-dvd/iso-image.nix b/nixos/modules/installer/cd-dvd/iso-image.nix
index 8f17e720aca1..d9d7254aba25 100644
--- a/nixos/modules/installer/cd-dvd/iso-image.nix
+++ b/nixos/modules/installer/cd-dvd/iso-image.nix
@@ -7,53 +7,65 @@
 with lib;
 
 let
-
-  # The Grub image.
-  grubImage = pkgs.runCommand "grub_eltorito" {}
+  # Timeout in syslinux is in units of 1/10 of a second.
+  # 0 is used to disable timeouts.
+  syslinuxTimeout = if config.boot.loader.timeout == null then
+      0
+    else
+      max (config.boot.loader.timeout * 10) 1;
+
+
+  max = x: y: if x > y then x else y;
+
+  # The configuration file for syslinux.
+
+  # Notes on syslinux configuration and UNetbootin compatiblity:
+  #   * Do not use '/syslinux/syslinux.cfg' as the path for this
+  #     configuration. UNetbootin will not parse the file and use it as-is.
+  #     This results in a broken configuration if the partition label does
+  #     not match the specified config.isoImage.volumeID. For this reason
+  #     we're using '/isolinux/isolinux.cfg'.
+  #   * Use APPEND instead of adding command-line arguments directly after
+  #     the LINUX entries.
+  #   * COM32 entries (chainload, reboot, poweroff) are not recognized. They
+  #     result in incorrect boot entries.
+
+  baseIsolinuxCfg =
     ''
-      ${pkgs.grub2}/bin/grub-mkimage -p /boot/grub -O i386-pc -o tmp biosdisk iso9660 help linux linux16 chain png jpeg echo gfxmenu reboot
-      cat ${pkgs.grub2}/lib/grub/*/cdboot.img tmp > $out
-    ''; # */
-
-
-  # The configuration file for Grub.
-  grubCfg =
-    ''
-      set default=${builtins.toString config.boot.loader.grub.default}
-      set timeout=${builtins.toString config.boot.loader.grub.timeout}
-
-      if loadfont /boot/grub/unicode.pf2; then
-        set gfxmode=640x480
-        insmod gfxterm
-        insmod vbe
-        terminal_output gfxterm
-
-        insmod png
-        if background_image /boot/grub/splash.png; then
-          set color_normal=white/black
-          set color_highlight=black/white
-        else
-          set menu_color_normal=cyan/blue
-          set menu_color_highlight=white/blue
-        fi
-
-      fi
-
-      ${config.boot.loader.grub.extraEntries}
+    SERIAL 0 38400
+    TIMEOUT ${builtins.toString syslinuxTimeout}
+    UI vesamenu.c32
+    MENU TITLE NixOS
+    MENU BACKGROUND /isolinux/background.png
+    DEFAULT boot
+
+    LABEL boot
+    MENU LABEL NixOS ${config.system.nixosVersion} Installer
+    LINUX /boot/bzImage
+    APPEND init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}
+    INITRD /boot/initrd
     '';
 
+  isolinuxMemtest86Entry = ''
+    LABEL memtest
+    MENU LABEL Memtest86+
+    LINUX /boot/memtest.bin
+    APPEND ${toString config.boot.loader.grub.memtest86.params}
+  '';
+
+  isolinuxCfg = baseIsolinuxCfg + (optionalString config.boot.loader.grub.memtest86.enable isolinuxMemtest86Entry);
 
   # The efi boot image
   efiDir = pkgs.runCommand "efi-directory" {} ''
-    mkdir -p $out/efi/boot
-    cp -v ${pkgs.gummiboot}/lib/gummiboot/gummiboot${targetArch}.efi $out/efi/boot/boot${targetArch}.efi
+    mkdir -p $out/EFI/boot
+    cp -v ${pkgs.gummiboot}/lib/gummiboot/gummiboot${targetArch}.efi $out/EFI/boot/boot${targetArch}.efi
     mkdir -p $out/loader/entries
     echo "title NixOS LiveCD" > $out/loader/entries/nixos-livecd.conf
     echo "linux /boot/bzImage" >> $out/loader/entries/nixos-livecd.conf
     echo "initrd /boot/initrd" >> $out/loader/entries/nixos-livecd.conf
     echo "options init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}" >> $out/loader/entries/nixos-livecd.conf
     echo "default nixos-livecd" > $out/loader/loader.conf
-    echo "timeout 5" >> $out/loader/loader.conf
+    echo "timeout ${builtins.toString config.boot.loader.gummiboot.timeout}" >> $out/loader/loader.conf
   '';
 
   efiImg = pkgs.runCommand "efi-image_eltorito" { buildInputs = [ pkgs.mtools pkgs.libfaketime ]; }
@@ -163,6 +175,22 @@ in
       '';
     };
 
+    isoImage.makeUsbBootable = mkOption {
+      default = false;
+      description = ''
+        Whether the ISO image should be bootable from CD as well as USB.
+      '';
+    };
+
+    isoImage.splashImage = mkOption {
+      default = pkgs.fetchurl {
+          url = https://raw.githubusercontent.com/NixOS/nixos-artwork/5729ab16c6a5793c10a2913b5a1b3f59b91c36ee/ideas/grub-splash/grub-nixos-1.png;
+          sha256 = "43fd8ad5decf6c23c87e9026170a13588c2eba249d9013cb9f888da5e2002217";
+        };
+      description = ''
+        The splash image to use in the bootloader.
+      '';
+    };
 
   };
 
@@ -176,7 +204,7 @@ in
 
     # !!! Hack - attributes expected by other modules.
     system.boot.loader.kernelFile = "bzImage";
-    environment.systemPackages = [ pkgs.grub2 ];
+    environment.systemPackages = [ pkgs.grub2 pkgs.syslinux ];
 
     # In stage 1 of the boot, mount the CD as the root FS by label so
     # that we don't need to know its device.  We pass the label of the
@@ -226,7 +254,7 @@ in
         options = "allow_other,cow,nonempty,chroot=/mnt-root,max_files=32768,hide_meta_files,dirs=/nix/.rw-store=rw:/nix/.ro-store=ro";
       };
 
-    boot.initrd.availableKernelModules = [ "squashfs" "iso9660" ];
+    boot.initrd.availableKernelModules = [ "squashfs" "iso9660" "usb-storage" ];
 
     boot.initrd.kernelModules = [ "loop" ];
 
@@ -246,15 +274,12 @@ in
     # Individual files to be included on the CD, outside of the Nix
     # store on the CD.
     isoImage.contents =
-      [ { source = grubImage;
-          target = "/boot/grub/grub_eltorito";
-        }
-        { source = pkgs.substituteAll  {
-            name = "grub.cfg";
-            src = pkgs.writeText "grub.cfg-in" grubCfg;
+      [ { source = pkgs.substituteAll  {
+            name = "isolinux.cfg";
+            src = pkgs.writeText "isolinux.cfg-in" isolinuxCfg;
             bootRoot = "/boot";
           };
-          target = "/boot/grub/grub.cfg";
+          target = "/isolinux/isolinux.cfg";
         }
         { source = config.boot.kernelPackages.kernel + "/bzImage";
           target = "/boot/bzImage";
@@ -262,51 +287,44 @@ in
         { source = config.system.build.initialRamdisk + "/initrd";
           target = "/boot/initrd";
         }
-        { source = "${pkgs.grub2}/share/grub/unicode.pf2";
-          target = "/boot/grub/unicode.pf2";
-        }
-        { source = config.boot.loader.grub.splashImage;
-          target = "/boot/grub/splash.png";
-        }
         { source = config.system.build.squashfsStore;
           target = "/nix-store.squashfs";
         }
+        { source = "${pkgs.syslinux}/share/syslinux";
+          target = "/isolinux";
+        }
+        { source = config.isoImage.splashImage;
+          target = "/isolinux/background.png";
+        }
       ] ++ optionals config.isoImage.makeEfiBootable [
         { source = efiImg;
           target = "/boot/efi.img";
         }
-        { source = "${efiDir}/efi";
-          target = "/efi";
+        { source = "${efiDir}/EFI";
+          target = "/EFI";
         }
         { source = "${efiDir}/loader";
           target = "/loader";
         }
-      ] ++ mapAttrsToList (n: v: { source = v; target = "/boot/${n}"; }) config.boot.loader.grub.extraFiles;
-
-    # The Grub menu.
-    boot.loader.grub.extraEntries =
-      ''
-        menuentry "NixOS ${config.system.nixosVersion} Installer" {
-          linux /boot/bzImage init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}
-          initrd /boot/initrd
-        }
-
-        menuentry "Boot from hard disk" {
-          set root=(hd0)
-          chainloader +1
+      ] ++ optionals config.boot.loader.grub.memtest86.enable [
+        { source = "${pkgs.memtest86plus}/memtest.bin";
+          target = "/boot/memtest.bin";
         }
-      '';
+      ];
 
-    boot.loader.grub.timeout = 10;
+    boot.loader.timeout = 10;
 
     # Create the ISO image.
     system.build.isoImage = import ../../../lib/make-iso9660-image.nix ({
-      inherit (pkgs) stdenv perl cdrkit pathsFromGraph;
+      inherit (pkgs) stdenv perl pathsFromGraph xorriso syslinux;
 
       inherit (config.isoImage) isoName compressImage volumeID contents;
 
       bootable = true;
-      bootImage = "/boot/grub/grub_eltorito";
+      bootImage = "/isolinux/isolinux.bin";
+    } // optionalAttrs config.isoImage.makeUsbBootable {
+      usbBootable = true;
+      isohybridMbrImage = "${pkgs.syslinux}/share/syslinux/isohdpfx.bin";
     } // optionalAttrs config.isoImage.makeEfiBootable {
       efiBootable = true;
       efiBootImage = "boot/efi.img";
diff --git a/nixos/release.nix b/nixos/release.nix
index f84501d741a6..e59b24d524ae 100644
--- a/nixos/release.nix
+++ b/nixos/release.nix
@@ -310,6 +310,10 @@ in rec {
   tests.udisks2 = callTest tests/udisks2.nix {};
   tests.virtualbox = callTest tests/virtualbox.nix {};
   tests.xfce = callTest tests/xfce.nix {};
+  tests.bootBiosCdrom = forAllSystems (system: scrubDrv (import tests/boot.nix { inherit system; }).bootBiosCdrom);
+  tests.bootBiosUsb = forAllSystems (system: scrubDrv (import tests/boot.nix { inherit system; }).bootBiosUsb);
+  tests.bootUefiCdrom = forAllSystems (system: scrubDrv (import tests/boot.nix { inherit system; }).bootUefiCdrom);
+  tests.bootUefiUsb = forAllSystems (system: scrubDrv (import tests/boot.nix { inherit system; }).bootUefiUsb);
 
 
   /* Build a bunch of typical closures so that Hydra can keep track of
diff --git a/nixos/tests/boot.nix b/nixos/tests/boot.nix
new file mode 100644
index 000000000000..2fdbb0c00b85
--- /dev/null
+++ b/nixos/tests/boot.nix
@@ -0,0 +1,63 @@
+{ system ? builtins.currentSystem }:
+
+with import ../lib/testing.nix { inherit system; };
+with import ../lib/qemu-flags.nix;
+with pkgs.lib;
+
+let
+
+  iso =
+    (import ../lib/eval-config.nix {
+      inherit system;
+      modules =
+        [ ../modules/installer/cd-dvd/installation-cd-minimal.nix
+          ../modules/testing/test-instrumentation.nix
+          { key = "serial";
+            boot.loader.grub.timeout = mkOverride 0 0;
+
+            # The test cannot access the network, so any sources we
+            # need must be included in the ISO.
+            isoImage.storeContents =
+              [ pkgs.glibcLocales
+                pkgs.sudo
+                pkgs.docbook5
+                pkgs.docbook5_xsl
+                pkgs.grub
+                pkgs.perlPackages.XMLLibXML
+                pkgs.unionfs-fuse
+                pkgs.gummiboot
+              ];
+          }
+        ];
+    }).config.system.build.isoImage;
+
+  makeBootTest = name: machineConfig:
+    makeTest {
+      inherit iso;
+      name = "boot-" + name;
+      nodes = { };
+      testScript =
+        ''
+          my $machine = createMachine({ ${machineConfig}, qemuFlags => '-m 768' });
+          $machine->start;
+          $machine->waitForUnit("multi-user.target");
+          $machine->shutdown;
+        '';
+    };
+in {
+    bootBiosCdrom = makeBootTest "bios-cdrom" ''
+        cdrom => glob("${iso}/iso/*.iso")
+      '';
+    bootBiosUsb = makeBootTest "bios-usb" ''
+        usb => glob("${iso}/iso/*.iso")
+      '';
+    bootUefiCdrom = makeBootTest "uefi-cdrom" ''
+        cdrom => glob("${iso}/iso/*.iso"),
+        bios => '${pkgs.OVMF}/FV/OVMF.fd'
+      '';
+    bootUefiUsb = makeBootTest "uefi-usb" ''
+        usb => glob("${iso}/iso/*.iso"),
+        bios => '${pkgs.OVMF}/FV/OVMF.fd'
+      '';
+  }
+