minimal-bootstrap.mes: Parallelize

5 files changed, 169 insertions, 129 deletions

diff --git a/pkgs/os-specific/linux/minimal-bootstrap/default.nix b/pkgs/os-specific/linux/minimal-bootstrap/default.nix
index 6c14f7fd6888..4a5c9b380588 100644
--- a/pkgs/os-specific/linux/minimal-bootstrap/default.nix
+++ b/pkgs/os-specific/linux/minimal-bootstrap/default.nix
@@ -30,6 +30,7 @@ lib.makeScope
     inherit (callPackage ./utils.nix { }) fetchurl derivationWithMeta writeTextFile writeText;
 
     test = kaem.runCommand "minimal-bootstrap-test" {} ''
+      echo ${mes.tests.get-version}
       echo ${tinycc-mes.compiler.tests.chain}
       mkdir ''${out}
     '';
diff --git a/pkgs/os-specific/linux/minimal-bootstrap/ln-boot/default.nix b/pkgs/os-specific/linux/minimal-bootstrap/ln-boot/default.nix
index 3ccc3042fd76..d9649bb85a24 100644
--- a/pkgs/os-specific/linux/minimal-bootstrap/ln-boot/default.nix
+++ b/pkgs/os-specific/linux/minimal-bootstrap/ln-boot/default.nix
@@ -21,6 +21,7 @@ kaem.runCommand "${pname}-${version}" {
 } ''
   mkdir -p ''${out}/bin
   ${mes}/bin/mes --no-auto-compile -e main ${mes}/bin/mescc.scm -- \
+    -L ${mes}/lib \
     -lc+tcc \
     -o ''${out}/bin/ln \
     ${src}
diff --git a/pkgs/os-specific/linux/minimal-bootstrap/mes/default.nix b/pkgs/os-specific/linux/minimal-bootstrap/mes/default.nix
index 23257ca44282..9add682eb66c 100644
--- a/pkgs/os-specific/linux/minimal-bootstrap/mes/default.nix
+++ b/pkgs/os-specific/linux/minimal-bootstrap/mes/default.nix
@@ -6,6 +6,12 @@
 , m2libc
 , mescc-tools
 }:
+
+# Maintenance note:
+# Build steps have been adapted from build-aux/bootstrap.sh.in
+# as well as the live-bootstrap project
+# https://github.com/fosslinux/live-bootstrap/blob/1bc4296091c51f53a5598050c8956d16e945b0f5/sysa/mes-0.24.2/mes-0.24.2.kaem
+
 let
   pname = "mes";
   version = "0.24.2";
@@ -28,162 +34,195 @@ let
   # add symlink() to libc+tcc so we can use it in ln-boot
   libc_tcc_SOURCES = sources.libc_tcc_SOURCES ++ [ "lib/linux/symlink.c" ];
 
-  compile = sources:
-    lib.concatMapStringsSep
-      "\n"
-      (f: ''CC -c ''${MES_PREFIX}/${f}'')
-      sources;
+  meta = with lib; {
+    description = "Scheme interpreter and C compiler for bootstrapping";
+    homepage = "https://www.gnu.org/software/mes";
+    license = licenses.gpl3Plus;
+    maintainers = with maintainers; [ emilytrau ];
+    platforms = [ "i686-linux" ];
+  };
+
+  srcPost = kaem.runCommand "${pname}-src-${version}" {
+    outputs = [ "out" "bin" ];
+    inherit meta;
+  } ''
+    # Unpack source
+    ungz --file ${src} --output mes.tar
+    mkdir ''${out}
+    cd ''${out}
+    untar --non-strict --file ''${NIX_BUILD_TOP}/mes.tar # ignore symlinks
+
+    MES_PREFIX=''${out}/mes-${version}
+
+    cd ''${MES_PREFIX}
+
+    cp ${config_h} include/mes/config.h
+
+    mkdir include/arch
+    cp include/linux/x86/syscall.h include/arch/syscall.h
+    cp include/linux/x86/kernel-stat.h include/arch/kernel-stat.h
+
+    # Remove pregenerated files
+    rm mes/module/mes/psyntax.pp mes/module/mes/psyntax.pp.header
+
+    # These files are symlinked in the repo
+    cp mes/module/srfi/srfi-9-struct.mes mes/module/srfi/srfi-9.mes
+    cp mes/module/srfi/srfi-9/gnu-struct.mes mes/module/srfi/srfi-9/gnu.mes
+
+    # Fixes to support newer M2-Planet
+    catm x86_defs.M1 ${m2libc}/x86/x86_defs.M1 lib/m2/x86/x86_defs.M1
+    cp x86_defs.M1 lib/m2/x86/x86_defs.M1
+    rm x86_defs.M1
+
+    # Remove environment impurities
+    __GUILE_LOAD_PATH="\"''${MES_PREFIX}/mes/module:''${MES_PREFIX}/module:${nyacc.guilePath}\""
+    boot0_scm=mes/module/mes/boot-0.scm
+    guile_mes=mes/module/mes/guile.mes
+    replace --file ''${boot0_scm} --output ''${boot0_scm} --match-on "(getenv \"GUILE_LOAD_PATH\")" --replace-with ''${__GUILE_LOAD_PATH}
+    replace --file ''${guile_mes} --output ''${guile_mes} --match-on "(getenv \"GUILE_LOAD_PATH\")" --replace-with ''${__GUILE_LOAD_PATH}
+
+    module_mescc_scm=module/mescc/mescc.scm
+    replace --file ''${module_mescc_scm} --output ''${module_mescc_scm} --match-on "(getenv \"M1\")" --replace-with "\"${mescc-tools}/bin/M1\""
+    replace --file ''${module_mescc_scm} --output ''${module_mescc_scm} --match-on "(getenv \"HEX2\")" --replace-with "\"${mescc-tools}/bin/hex2\""
+    replace --file ''${module_mescc_scm} --output ''${module_mescc_scm} --match-on "(getenv \"BLOOD_ELF\")" --replace-with "\"${mescc-tools}/bin/blood-elf\""
+    replace --file ''${module_mescc_scm} --output ''${module_mescc_scm} --match-on "(getenv \"srcdest\")" --replace-with "\"''${MES_PREFIX}\""
+
+    mes_c=src/mes.c
+    replace --file ''${mes_c} --output ''${mes_c} --match-on "getenv (\"MES_PREFIX\")" --replace-with "\"''${MES_PREFIX}\""
+    replace --file ''${mes_c} --output ''${mes_c} --match-on "getenv (\"srcdest\")" --replace-with "\"''${MES_PREFIX}\""
+
+    # Increase runtime resource limits
+    gc_c=src/gc.c
+    replace --file ''${gc_c} --output ''${gc_c} --match-on "getenv (\"MES_ARENA\")" --replace-with "\"100000000\""
+    replace --file ''${gc_c} --output ''${gc_c} --match-on "getenv (\"MES_MAX_ARENA\")" --replace-with "\"100000000\""
+    replace --file ''${gc_c} --output ''${gc_c} --match-on "getenv (\"MES_STACK\")" --replace-with "\"6000000\""
+
+    # Create mescc.scm
+    mescc_in=scripts/mescc.scm.in
+    replace --file ''${mescc_in} --output ''${mescc_in} --match-on "(getenv \"MES_PREFIX\")" --replace-with "\"''${MES_PREFIX}\""
+    replace --file ''${mescc_in} --output ''${mescc_in} --match-on "(getenv \"includedir\")" --replace-with "\"''${MES_PREFIX}/include\""
+    replace --file ''${mescc_in} --output ''${mescc_in} --match-on "(getenv \"libdir\")" --replace-with "\"''${MES_PREFIX}/lib\""
+    replace --file ''${mescc_in} --output ''${mescc_in} --match-on @prefix@ --replace-with ''${MES_PREFIX}
+    replace --file ''${mescc_in} --output ''${mescc_in} --match-on @VERSION@ --replace-with ${version}
+    replace --file ''${mescc_in} --output ''${mescc_in} --match-on @mes_cpu@ --replace-with x86
+    replace --file ''${mescc_in} --output ''${mescc_in} --match-on @mes_kernel@ --replace-with linux
+    mkdir -p ''${bin}/bin
+    cp ''${mescc_in} ''${bin}/bin/mescc.scm
+
+    # Build mes-m2
+    mes_cpu=x86
+    stage0_cpu=x86
+    kaem --verbose --strict --file kaem.run
+    cp bin/mes-m2 ''${bin}/bin/mes-m2
+    chmod 555 ''${bin}/bin/mes-m2
+  '';
+
+  srcPrefix = "${srcPost.out}/mes-${version}";
+
+  cc = "${srcPost.bin}/bin/mes-m2";
+  ccArgs = [
+    "-e" "main"
+    "${srcPost.bin}/bin/mescc.scm"
+    "--"
+    "-D" "HAVE_CONFIG_H=1"
+    "-I" "${srcPrefix}/include"
+    "-I" "${srcPrefix}/include/linux/x86"
+  ];
+
+  CC = toString ([ cc ] ++ ccArgs);
+
   replaceExt = ext: source:
     lib.replaceStrings
       [ ".c" ]
       [ ext ]
       (builtins.baseNameOf source);
+
+  compile = source: kaem.runCommand (replaceExt "" source) {} ''
+    mkdir ''${out}
+    cd ''${out}
+    ${CC} -c ${srcPrefix}/${source}
+  '';
+
+  crt1 = compile "/lib/linux/x86-mes-mescc/crt1.c";
+
+  getRes = suffix: res: "${res}/${res.name}${suffix}";
+
   archive = out: sources:
-    "catm ${out} ${lib.concatMapStringsSep " " (replaceExt ".o") sources}";
+    "catm ${out} ${lib.concatMapStringsSep " " (getRes ".o") sources}";
   sourceArchive = out: sources:
-    "catm ${out} ${lib.concatMapStringsSep " " (replaceExt ".s") sources}";
-in
-kaem.runCommand "${pname}-${version}" {
+    "catm ${out} ${lib.concatMapStringsSep " " (getRes ".s") sources}";
+
+  mkLib = libname: sources: let
+    os = map compile sources;
+  in kaem.runCommand "${pname}-${libname}-${version}" {
+    inherit meta;
+  } ''
+    LIBDIR=''${out}/lib
+    mkdir -p ''${LIBDIR}
+    cd ''${LIBDIR}
+
+    ${archive "${libname}.a" os}
+    ${sourceArchive "${libname}.s" os}
+  '';
+
+  libc-mini = mkLib "libc-mini" libc_mini_SOURCES;
+  libmescc = mkLib "libmescc" libmescc_SOURCES;
+  libc = mkLib "libc" libc_SOURCES;
+  libc_tcc = mkLib "libc+tcc" libc_tcc_SOURCES;
+
+in kaem.runCommand "${pname}-${version}" {
   inherit pname version;
 
-  passthru = { inherit src nyacc; };
+  passthru = { inherit src srcPost srcPrefix nyacc; };
+  passthru.tests.get-version = result: kaem.runCommand "${pname}-get-version-${version}" {} ''
+    ${result}/bin/mes --version
+    mkdir ''${out}
+  '';
 
-  meta = with lib; {
-    description = "Scheme interpreter and C compiler for bootstrapping";
-    homepage = "https://www.gnu.org/software/mes";
-    license = licenses.gpl3Plus;
-    maintainers = with maintainers; [ emilytrau ];
-    platforms = [ "i686-linux" ];
-  };
+  inherit meta;
 }
-# Maintenance note:
-# Build steps have been adapted from build-aux/bootstrap.sh.in
-# as well as the live-bootstrap project
-# https://github.com/fosslinux/live-bootstrap/blob/1bc4296091c51f53a5598050c8956d16e945b0f5/sysa/mes-0.24.2/mes-0.24.2.kaem
 ''
-  # Unpack source
-  ungz --file ${src} --output mes.tar
-  mkdir ''${out} ''${out}/bin ''${out}/share
-  cd ''${out}/share
-  untar --non-strict --file ''${NIX_BUILD_TOP}/mes.tar # ignore symlinks
-
-  MES_PREFIX=''${out}/share/mes-${version}
-  LIBDIR=''${MES_PREFIX}/lib
-
-  cd ''${MES_PREFIX}
-
-  cp ${config_h} include/mes/config.h
-
-  mkdir include/arch
-  cp include/linux/x86/syscall.h include/arch/syscall.h
-  cp include/linux/x86/kernel-stat.h include/arch/kernel-stat.h
-
-  # Remove pregenerated files
-  rm mes/module/mes/psyntax.pp mes/module/mes/psyntax.pp.header
-
-  # These files are symlinked in the repo
-  cp mes/module/srfi/srfi-9-struct.mes mes/module/srfi/srfi-9.mes
-  cp mes/module/srfi/srfi-9/gnu-struct.mes mes/module/srfi/srfi-9/gnu.mes
-
-  # Fixes to support newer M2-Planet
-  catm x86_defs.M1 ${m2libc}/x86/x86_defs.M1 lib/m2/x86/x86_defs.M1
-  cp x86_defs.M1 lib/m2/x86/x86_defs.M1
-  rm x86_defs.M1
-
-  # Remove environment impurities
-  __GUILE_LOAD_PATH="\"''${MES_PREFIX}/mes/module:''${MES_PREFIX}/module:${nyacc.guilePath}\""
-  boot0_scm=mes/module/mes/boot-0.scm
-  guile_mes=mes/module/mes/guile.mes
-  replace --file ''${boot0_scm} --output ''${boot0_scm} --match-on "(getenv \"GUILE_LOAD_PATH\")" --replace-with ''${__GUILE_LOAD_PATH}
-  replace --file ''${guile_mes} --output ''${guile_mes} --match-on "(getenv \"GUILE_LOAD_PATH\")" --replace-with ''${__GUILE_LOAD_PATH}
-
-  module_mescc_scm=module/mescc/mescc.scm
-  replace --file ''${module_mescc_scm} --output ''${module_mescc_scm} --match-on "(getenv \"M1\")" --replace-with "\"${mescc-tools}/bin/M1\""
-  replace --file ''${module_mescc_scm} --output ''${module_mescc_scm} --match-on "(getenv \"HEX2\")" --replace-with "\"${mescc-tools}/bin/hex2\""
-  replace --file ''${module_mescc_scm} --output ''${module_mescc_scm} --match-on "(getenv \"BLOOD_ELF\")" --replace-with "\"${mescc-tools}/bin/blood-elf\""
-  replace --file ''${module_mescc_scm} --output ''${module_mescc_scm} --match-on "(getenv \"srcdest\")" --replace-with "\"''${MES_PREFIX}\""
-
-  mes_c=src/mes.c
-  replace --file ''${mes_c} --output ''${mes_c} --match-on "getenv (\"MES_PREFIX\")" --replace-with "\"''${MES_PREFIX}\""
-  replace --file ''${mes_c} --output ''${mes_c} --match-on "getenv (\"srcdest\")" --replace-with "\"''${MES_PREFIX}\""
-
-  # Increase runtime resource limits
-  gc_c=src/gc.c
-  replace --file ''${gc_c} --output ''${gc_c} --match-on "getenv (\"MES_ARENA\")" --replace-with "\"100000000\""
-  replace --file ''${gc_c} --output ''${gc_c} --match-on "getenv (\"MES_MAX_ARENA\")" --replace-with "\"100000000\""
-  replace --file ''${gc_c} --output ''${gc_c} --match-on "getenv (\"MES_STACK\")" --replace-with "\"6000000\""
-
-  # Create mescc.scm
-  mescc_in=scripts/mescc.scm.in
-  replace --file ''${mescc_in} --output ''${mescc_in} --match-on "(getenv \"MES_PREFIX\")" --replace-with "\"''${MES_PREFIX}\""
-  replace --file ''${mescc_in} --output ''${mescc_in} --match-on "(getenv \"includedir\")" --replace-with "\"''${MES_PREFIX}/include\""
-  replace --file ''${mescc_in} --output ''${mescc_in} --match-on "(getenv \"libdir\")" --replace-with "\"''${MES_PREFIX}/lib\""
-  replace --file ''${mescc_in} --output ''${mescc_in} --match-on @prefix@ --replace-with ''${MES_PREFIX}
-  replace --file ''${mescc_in} --output ''${mescc_in} --match-on @VERSION@ --replace-with ${version}
-  replace --file ''${mescc_in} --output ''${mescc_in} --match-on @mes_cpu@ --replace-with x86
-  replace --file ''${mescc_in} --output ''${mescc_in} --match-on @mes_kernel@ --replace-with linux
-  cp ''${mescc_in} ''${out}/bin/mescc.scm
-
-  # Build mes-m2
-  mes_cpu=x86
-  stage0_cpu=x86
-  kaem --verbose --strict --file kaem.run
-  cp bin/mes-m2 ''${out}/bin/mes-m2
-  chmod 555 ''${out}/bin/mes-m2
+  LIBDIR=''${out}/lib
+  BINDIR=''${out}/bin
 
+  mkdir -p ''${out} ''${LIBDIR} ''${BINDIR}
+
+  cp ${srcPost.bin}/bin/mescc.scm ''${BINDIR}/mescc.scm
+  cp ${srcPost.bin}/bin/mes-m2 ''${BINDIR}/mes-m2
+  chmod 555 ''${BINDIR}/mes-m2
 
   # Recompile Mes and Mes C library using mes-m2 bootstrapped Mes
-  cd ''${NIX_BUILD_TOP}
-  alias CC="''${out}/bin/mes-m2 -e main ''${out}/bin/mescc.scm -- -D HAVE_CONFIG_H=1 -I ''${MES_PREFIX}/include -I ''${MES_PREFIX}/include/linux/x86"
   mkdir -p ''${LIBDIR}/x86-mes
 
   # crt1.o
-  CC -c ''${MES_PREFIX}/lib/linux/x86-mes-mescc/crt1.c
-  cp crt1.o ''${LIBDIR}/x86-mes
-  cp crt1.s ''${LIBDIR}/x86-mes
+  cp ${crt1}/crt1.o ''${LIBDIR}/x86-mes
+  cp ${crt1}/crt1.s ''${LIBDIR}/x86-mes
 
   # libc-mini.a
-  ${compile libc_mini_SOURCES}
-  ${archive "libc-mini.a" libc_mini_SOURCES}
-  ${sourceArchive "libc-mini.s" libc_mini_SOURCES}
-  cp libc-mini.a ''${LIBDIR}/x86-mes
-  cp libc-mini.s ''${LIBDIR}/x86-mes
+  cp ${libc-mini}/lib/libc-mini.a ''${LIBDIR}/x86-mes
+  cp ${libc-mini}/lib/libc-mini.s ''${LIBDIR}/x86-mes
 
   # libmescc.a
-  ${compile libmescc_SOURCES}
-  ${archive "libmescc.a" libmescc_SOURCES}
-  ${sourceArchive "libmescc.s" libmescc_SOURCES}
-  cp libmescc.a ''${LIBDIR}/x86-mes
-  cp libmescc.s ''${LIBDIR}/x86-mes
+  cp ${libmescc}/lib/libmescc.a ''${LIBDIR}/x86-mes
+  cp ${libmescc}/lib/libmescc.s ''${LIBDIR}/x86-mes
 
   # libc.a
-  ${compile libc_SOURCES}
-  ${archive "libc.a" libc_SOURCES}
-  ${sourceArchive "libc.s" libc_SOURCES}
-  cp libc.a ''${LIBDIR}/x86-mes
-  cp libc.s ''${LIBDIR}/x86-mes
+  cp ${libc}/lib/libc.a ''${LIBDIR}/x86-mes
+  cp ${libc}/lib/libc.s ''${LIBDIR}/x86-mes
 
   # libc+tcc.a
-  # optimisation: don't recompile common libc sources
-  ${compile (lib.subtractLists libc_SOURCES libc_tcc_SOURCES)}
-  ${archive "libc+tcc.a" libc_tcc_SOURCES}
-  ${sourceArchive "libc+tcc.s" libc_tcc_SOURCES}
-  cp libc+tcc.a ''${LIBDIR}/x86-mes
-  cp libc+tcc.s ''${LIBDIR}/x86-mes
+  cp ${libc_tcc}/lib/libc+tcc.a ''${LIBDIR}/x86-mes
+  cp ${libc_tcc}/lib/libc+tcc.s ''${LIBDIR}/x86-mes
 
   # Build mes itself
-  ${compile mes_SOURCES}
   ''${out}/bin/mes-m2 -e main ''${out}/bin/mescc.scm -- \
     --base-address 0x08048000 \
-    -L ''${MES_PREFIX}/lib \
-    -L . \
+    -L ''${srcPrefix}/lib \
+    -L ''${LIBDIR} \
     -lc \
     -lmescc \
     -nostdlib \
     -o ''${out}/bin/mes \
-    crt1.o \
-    ${lib.concatMapStringsSep " " (replaceExt ".o") mes_SOURCES}
-
-  # Check
-  ''${out}/bin/mes --version
+    ''${LIBDIR}/x86-mes/crt1.o \
+    ${lib.concatMapStringsSep " " (getRes ".o") (map compile mes_SOURCES)}
 ''
diff --git a/pkgs/os-specific/linux/minimal-bootstrap/mes/libc.nix b/pkgs/os-specific/linux/minimal-bootstrap/mes/libc.nix
index d036cb10125d..28aa2a978f64 100644
--- a/pkgs/os-specific/linux/minimal-bootstrap/mes/libc.nix
+++ b/pkgs/os-specific/linux/minimal-bootstrap/mes/libc.nix
@@ -11,8 +11,6 @@ let
   sources = (import ./sources.nix).x86.linux.gcc;
   inherit (sources) libtcc1_SOURCES libc_gnu_SOURCES;
 
-  prefix = "${mes}/share/mes-${version}";
-
   # Concatenate all source files into a convenient bundle
   # "gcc" variants of source files (eg. "lib/linux/x86-mes-gcc") can also be
   # compiled by tinycc
@@ -37,11 +35,11 @@ kaem.runCommand "${pname}-${version}" {
     platforms = [ "i686-linux" ];
   };
 } ''
-  cd ${prefix}
+  cd ${mes.srcPrefix}
 
   # mescc compiled libc.a
   mkdir -p ''${out}/lib/x86-mes
-  cp lib/x86-mes/libc.a ''${out}/lib/x86-mes
+  cp ${mes}/lib/x86-mes/libc.a ''${out}/lib/x86-mes
 
   # libc.c
   catm ''${TMPDIR}/first.c ${lib.concatStringsSep " " firstLibc}
@@ -59,5 +57,5 @@ kaem.runCommand "${pname}-${version}" {
   cp lib/posix/getopt.c ''${out}/lib/libgetopt.c
 
   # Install headers
-  ln -s ${prefix}/include ''${out}/include
+  ln -s ${mes.srcPrefix}/include ''${out}/include
 ''
diff --git a/pkgs/os-specific/linux/minimal-bootstrap/tinycc/bootstrappable.nix b/pkgs/os-specific/linux/minimal-bootstrap/tinycc/bootstrappable.nix
index 93ff671e1efe..86bcf18643c9 100644
--- a/pkgs/os-specific/linux/minimal-bootstrap/tinycc/bootstrappable.nix
+++ b/pkgs/os-specific/linux/minimal-bootstrap/tinycc/bootstrappable.nix
@@ -74,6 +74,7 @@ let
         ${src}/tcc.c
       mkdir -p ''${out}/bin
       ${mes}/bin/mes --no-auto-compile -e main ${mes}/bin/mescc.scm -- \
+        -L ${mes}/lib \
         -l c+tcc \
         -o ''${out}/bin/tcc \
         tcc.s