diff options
Diffstat (limited to 'nixpkgs/pkgs/applications/virtualization/singularity')
3 files changed, 495 insertions, 0 deletions
diff --git a/nixpkgs/pkgs/applications/virtualization/singularity/apptainer/0001-ldCache-patch-for-driverLink.patch b/nixpkgs/pkgs/applications/virtualization/singularity/apptainer/0001-ldCache-patch-for-driverLink.patch new file mode 100644 index 000000000000..c931894bc21f --- /dev/null +++ b/nixpkgs/pkgs/applications/virtualization/singularity/apptainer/0001-ldCache-patch-for-driverLink.patch @@ -0,0 +1,84 @@ +From 783ec26c0d83013baf04579a6a415d7f8776ac93 Mon Sep 17 00:00:00 2001 +From: Someone Serge <sergei.kozlukov@aalto.fi> +Date: Sun, 7 Jan 2024 11:48:24 +0000 +Subject: [PATCH] ldCache(): patch for @driverLink@ + +--- + internal/pkg/util/paths/resolve.go | 41 +++++++++++++++++++++++++++--- + 1 file changed, 38 insertions(+), 3 deletions(-) + +diff --git a/internal/pkg/util/paths/resolve.go b/internal/pkg/util/paths/resolve.go +index db45d9db1..9d0110b6b 100644 +--- a/internal/pkg/util/paths/resolve.go ++++ b/internal/pkg/util/paths/resolve.go +@@ -14,6 +14,7 @@ import ( + "fmt" + "os" + "os/exec" ++ "path" + "path/filepath" + "regexp" + "strings" +@@ -154,14 +155,49 @@ func Resolve(fileList []string) ([]string, []string, error) { + // lists three variants of libEGL.so.1 that are in different locations, we only + // report the first, highest priority, variant. + func ldCache() (map[string]string, error) { ++ driverDirs := strings.Split("@driverLink@/lib", ":") ++ if machine, err := elfMachine(); err == nil && machine == elf.EM_386 { ++ driverDirs = strings.Split("@driverLink@-32/lib", ":") ++ } ++ ++ soPattern, err := regexp.Compile(`[^\s]+\.so(\.\d+(\.\d+(\.\d+)?)?)?$`) ++ if err != nil { ++ return nil, fmt.Errorf("could not compile ldconfig regexp: %v", err) ++ } ++ ++ ldCache := make(map[string]string) ++ for _, dirPath := range driverDirs { ++ dir, err := os.Open(dirPath) ++ if err != nil { ++ /* Maybe we're not running under NixOS */ ++ continue ++ } ++ files, err := dir.ReadDir(-1) ++ if err != nil { ++ continue ++ } ++ for _, f := range files { ++ if !soPattern.MatchString(f.Name()) { ++ continue ++ } ++ libName := f.Name() ++ libPath := path.Join(dirPath, f.Name()) ++ if _, ok := ldCache[libName]; !ok { ++ ldCache[libName] = libPath ++ } ++ } ++ } ++ + // walk through the ldconfig output and add entries which contain the filenames + // returned by nvidia-container-cli OR the nvliblist.conf file contents + ldconfig, err := bin.FindBin("ldconfig") +- if err != nil { ++ if err != nil && len(ldCache) == 0 { ++ // Note that missing ldconfig is only an "error" as long ++ // as there's no driverLink + return nil, err + } + out, err := exec.Command(ldconfig, "-p").Output() +- if err != nil { ++ if err != nil && len(ldCache) == 0 { + return nil, fmt.Errorf("could not execute ldconfig: %v", err) + } + +@@ -173,7 +209,6 @@ func ldCache() (map[string]string, error) { + } + + // store library name with associated path +- ldCache := make(map[string]string) + for _, match := range r.FindAllSubmatch(out, -1) { + if match != nil { + // libName is the "libnvidia-ml.so.1" (from the above example) +-- +2.42.0 + diff --git a/nixpkgs/pkgs/applications/virtualization/singularity/generic.nix b/nixpkgs/pkgs/applications/virtualization/singularity/generic.nix new file mode 100644 index 000000000000..85992e2abce9 --- /dev/null +++ b/nixpkgs/pkgs/applications/virtualization/singularity/generic.nix @@ -0,0 +1,318 @@ +# Configurations that should only be overrided by +# overrideAttrs +{ pname +, version +, src +, projectName # "apptainer" or "singularity" +, vendorHash ? null +, deleteVendor ? false +, proxyVendor ? false +, extraConfigureFlags ? [ ] +, extraDescription ? "" +, extraMeta ? { } +}: + +let + # Workaround for vendor-related attributes not overridable (#86349) + # should be removed when the issue is resolved + _defaultGoVendorArgs = { + inherit + vendorHash + deleteVendor + proxyVendor + ; + }; +in +{ lib +, buildGoModule +, runCommandLocal + # Native build inputs +, addDriverRunpath +, makeWrapper +, pkg-config +, util-linux +, which + # Build inputs +, bash +, callPackage +, conmon +, coreutils +, cryptsetup +, e2fsprogs +, fakeroot +, fuse2fs ? e2fsprogs.fuse2fs +, go +, gpgme +, libseccomp +, libuuid + # This is for nvidia-container-cli +, nvidia-docker +, openssl +, squashfsTools +, squashfuse + # Test dependencies +, singularity-tools +, cowsay +, hello + # Overridable configurations +, enableNvidiaContainerCli ? true + # --nvccli currently requires extra privileges: + # https://github.com/apptainer/apptainer/issues/1893#issuecomment-1881240800 +, forceNvcCli ? false + # Compile with seccomp support + # SingularityCE 3.10.0 and above requires explicit --without-seccomp when libseccomp is not available. +, enableSeccomp ? true + # Whether the configure script treat SUID support as default + # When equal to enableSuid, it supress the --with-suid / --without-suid build flag + # It can be set to `null` to always pass either --with-suid or --without-suided + # Type: null or boolean +, defaultToSuid ? true + # Whether to compile with SUID support +, enableSuid ? false +, starterSuidPath ? null +, substituteAll + # newuidmapPath and newgidmapPath are to support --fakeroot + # where those SUID-ed executables are unavailable from the FHS system PATH. + # Path to SUID-ed newuidmap executable +, newuidmapPath ? null + # Path to SUID-ed newgidmap executable +, newgidmapPath ? null + # External LOCALSTATEDIR +, externalLocalStateDir ? null + # Remove the symlinks to `singularity*` when projectName != "singularity" +, removeCompat ? false + # Workaround #86349 + # should be removed when the issue is resolved +, vendorHash ? _defaultGoVendorArgs.vendorHash +, deleteVendor ? _defaultGoVendorArgs.deleteVendor +, proxyVendor ? _defaultGoVendorArgs.proxyVendor +}: + +let + defaultPathOriginal = "/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin"; + privileged-un-utils = if ((newuidmapPath == null) && (newgidmapPath == null)) then null else + (runCommandLocal "privileged-un-utils" { } '' + mkdir -p "$out/bin" + ln -s ${lib.escapeShellArg newuidmapPath} "$out/bin/newuidmap" + ln -s ${lib.escapeShellArg newgidmapPath} "$out/bin/newgidmap" + ''); +in +(buildGoModule { + inherit pname version src; + + patches = lib.optionals (projectName == "apptainer") [ + (substituteAll { src = ./apptainer/0001-ldCache-patch-for-driverLink.patch; inherit (addDriverRunpath) driverLink; }) + ]; + + # Override vendorHash with the output got from + # nix-prefetch -E "{ sha256 }: ((import ./. { }).apptainer.override { vendorHash = sha256; }).goModules" + # or with `null` when using vendored source tarball. + inherit vendorHash deleteVendor proxyVendor; + + # go is used to compile extensions when building container images + allowGoReference = true; + + strictDeps = true; + + passthru = { + inherit + enableSeccomp + enableSuid + externalLocalStateDir + projectName + removeCompat + starterSuidPath + ; + }; + + nativeBuildInputs = [ + makeWrapper + pkg-config + util-linux + which + ]; + + # Search inside the project sources + # and see the `control` file of the Debian package from upstream repos + # for build-time dependencies and run-time utilities + # apptainer/apptainer: https://github.com/apptainer/apptainer/blob/main/dist/debian/control + # sylabs/singularity: https://github.com/sylabs/singularity/blob/main/debian/control + + buildInputs = [ + bash # To patch /bin/sh shebangs. + conmon + cryptsetup + gpgme + libuuid + openssl + squashfsTools # Required at build time by SingularityCE + ] + ++ lib.optional enableNvidiaContainerCli nvidia-docker + ++ lib.optional enableSeccomp libseccomp + ; + + configureScript = "./mconfig"; + + configureFlags = [ + "--localstatedir=${if externalLocalStateDir != null then externalLocalStateDir else "${placeholder "out"}/var/lib"}" + "--runstatedir=/var/run" + ] + ++ lib.optional (!enableSeccomp) "--without-seccomp" + ++ lib.optional (enableSuid != defaultToSuid) (if enableSuid then "--with-suid" else "--without-suid") + ++ extraConfigureFlags + ; + + # causes redefinition of _FORTIFY_SOURCE + hardeningDisable = [ "fortify3" ]; + + # Packages to prefix to the Apptainer/Singularity container runtime default PATH + # Use overrideAttrs to override + defaultPathInputs = [ + bash + coreutils + cryptsetup # cryptsetup + fakeroot + fuse2fs # Mount ext3 filesystems + go + privileged-un-utils + squashfsTools # mksquashfs unsquashfs # Make / unpack squashfs image + squashfuse # squashfuse_ll squashfuse # Mount (without unpacking) a squashfs image without privileges + ] + ++ lib.optional enableNvidiaContainerCli nvidia-docker + ; + + postPatch = '' + if [[ ! -e .git || ! -e VERSION ]]; then + echo "${version}" > VERSION + fi + + # Patch shebangs for script run during build + patchShebangs --build "$configureScript" makeit e2e scripts mlocal/scripts + + # Patching the hard-coded defaultPath by prefixing the packages in defaultPathInputs + substituteInPlace cmd/internal/cli/actions.go \ + --replace "defaultPath = \"${defaultPathOriginal}\"" "defaultPath = \"''${defaultPathInputs// /\/bin:}''${defaultPathInputs:+/bin:}${defaultPathOriginal}\"" + + substituteInPlace internal/pkg/util/gpu/nvidia.go \ + --replace \ + 'return fmt.Errorf("/usr/bin not writable in the container")' \ + "" + ''; + + postConfigure = '' + # Code borrowed from pkgs/stdenv/generic/setup.sh configurePhase() + + # set to empty if unset + : ''${configureFlags=} + + # shellcheck disable=SC2086 + $configureScript -V ${version} "''${prefixKey:---prefix=}$prefix" $configureFlags "''${configureFlagsArray[@]}" + + # End of the code from pkgs/stdenv/generic/setup.sh configurPhase() + ''; + + buildPhase = '' + runHook preBuild + make -C builddir -j"$NIX_BUILD_CORES" + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + make -C builddir install LOCALSTATEDIR="$out/var/lib" + runHook postInstall + ''; + + postFixup = '' + substituteInPlace "$out/bin/run-singularity" \ + --replace "/usr/bin/env ${projectName}" "$out/bin/${projectName}" + wrapProgram "$out/bin/${projectName}" \ + --prefix PATH : "''${defaultPathInputs// /\/bin:}''${defaultPathInputs:+/bin:}" + # Make changes in the config file + ${lib.optionalString forceNvcCli '' + substituteInPlace "$out/etc/${projectName}/${projectName}.conf" \ + --replace "use nvidia-container-cli = no" "use nvidia-container-cli = yes" + ''} + ${lib.optionalString (enableNvidiaContainerCli && projectName == "singularity") '' + substituteInPlace "$out/etc/${projectName}/${projectName}.conf" \ + --replace "# nvidia-container-cli path =" "nvidia-container-cli path = ${nvidia-docker}/bin/nvidia-container-cli" + ''} + ${lib.optionalString (removeCompat && (projectName != "singularity")) '' + unlink "$out/bin/singularity" + for file in "$out"/share/man/man?/singularity*.gz; do + if [[ -L "$file" ]]; then + unlink "$file" + fi + done + for file in "$out"/share/*-completion/completions/singularity; do + if [[ -e "$file" ]] + rm "$file" + done + ''} + ${lib.optionalString enableSuid (lib.warnIf (starterSuidPath == null) "${projectName}: Null starterSuidPath when enableSuid produces non-SUID-ed starter-suid and run-time permission denial." '' + chmod +x $out/libexec/${projectName}/bin/starter-suid + '')} + ${lib.optionalString (enableSuid && (starterSuidPath != null)) '' + mv "$out"/libexec/${projectName}/bin/starter-suid{,.orig} + ln -s ${lib.escapeShellArg starterSuidPath} "$out/libexec/${projectName}/bin/starter-suid" + ''} + ''; + + meta = with lib; { + description = "Application containers for linux" + extraDescription; + longDescription = '' + Singularity (the upstream) renamed themselves to Apptainer + to distinguish themselves from a fork made by Sylabs Inc.. See + + https://sylabs.io/2021/05/singularity-community-edition + https://apptainer.org/news/community-announcement-20211130 + ''; + license = licenses.bsd3; + platforms = platforms.linux; + maintainers = with maintainers; [ jbedo ShamrockLee ]; + mainProgram = projectName; + } // extraMeta; +}).overrideAttrs (finalAttrs: prevAttrs: { + passthru = prevAttrs.passthru or { } // { + tests = { + image-hello-cowsay = singularity-tools.buildImage { + name = "hello-cowsay"; + contents = [ hello cowsay ]; + singularity = finalAttrs.finalPackage; + }; + }; + gpuChecks = lib.optionalAttrs (projectName == "apptainer") { + # Should be in tests, but Ofborg would skip image-hello-cowsay because + # saxpy is unfree. + image-saxpy = callPackage + ({ singularity-tools, cudaPackages }: + singularity-tools.buildImage { + name = "saxpy"; + contents = [ cudaPackages.saxpy ]; + memSize = 2048; + diskSize = 2048; + singularity = finalAttrs.finalPackage; + }) + { }; + saxpy = + callPackage + ({ runCommand, writeShellScriptBin }: + let + unwrapped = writeShellScriptBin "apptainer-cuda-saxpy" + '' + ${lib.getExe finalAttrs.finalPackage} exec --nv $@ ${finalAttrs.passthru.tests.image-saxpy} saxpy + ''; + in + runCommand "run-apptainer-cuda-saxpy" + { + requiredSystemFeatures = [ "cuda" ]; + nativeBuildInputs = [ unwrapped ]; + passthru = { inherit unwrapped; }; + } + '' + apptainer-cuda-saxpy + '') + { }; + }; + }; +}) diff --git a/nixpkgs/pkgs/applications/virtualization/singularity/packages.nix b/nixpkgs/pkgs/applications/virtualization/singularity/packages.nix new file mode 100644 index 000000000000..6d71d1abe16e --- /dev/null +++ b/nixpkgs/pkgs/applications/virtualization/singularity/packages.nix @@ -0,0 +1,93 @@ +{ callPackage +, fetchFromGitHub +, nixos +, conmon +}: +let + apptainer = callPackage + (import ./generic.nix rec { + pname = "apptainer"; + version = "1.2.5"; + projectName = "apptainer"; + + src = fetchFromGitHub { + owner = "apptainer"; + repo = "apptainer"; + rev = "refs/tags/v${version}"; + hash = "sha256-1XuqyNXyYrmIfqp8450z8+qET15hKVfj2v2iN9QPmDk="; + }; + + # Update by running + # nix-prefetch -E "{ sha256 }: ((import ./. { }).apptainer.override { vendorHash = sha256; }).goModules" + # at the root directory of the Nixpkgs repository + vendorHash = "sha256-Y0gOqg+WGgssXGEYHc9IFwiIpkb3hetlQI89vseAQPc="; + + extraDescription = " (previously known as Singularity)"; + extraMeta.homepage = "https://apptainer.org"; + }) + { + # Apptainer doesn't depend on conmon + conmon = null; + + # Apptainer builders require explicit --with-suid / --without-suid flag + # when building on a system with disabled unprivileged namespace. + # See https://github.com/NixOS/nixpkgs/pull/215690#issuecomment-1426954601 + defaultToSuid = null; + }; + + singularity = callPackage + (import ./generic.nix rec { + pname = "singularity-ce"; + version = "4.1.1"; + projectName = "singularity"; + + src = fetchFromGitHub { + owner = "sylabs"; + repo = "singularity"; + rev = "refs/tags/v${version}"; + hash = "sha256-BKuo+W75wsK8HFB+5CtKPqR4nDw167pAAiuISOjML7k="; + }; + + # Update by running + # nix-prefetch -E "{ sha256 }: ((import ./. { }).singularity.override { vendorHash = sha256; }).goModules" + # at the root directory of the Nixpkgs repository + vendorHash = "sha256-Hg32YtXUFQI7OslW3E3QpxCiypwaK8BDAl3YAM6kMnw="; + + # Do not build conmon and squashfuse from the Git submodule sources, + # Use Nixpkgs provided version + extraConfigureFlags = [ + "--without-conmon" + "--without-squashfuse" + ]; + + extraDescription = " (Sylabs Inc's fork of Singularity, a.k.a. SingularityCE)"; + extraMeta.homepage = "https://sylabs.io/"; + }) + { + defaultToSuid = true; + }; + + genOverridenNixos = package: packageName: (nixos { + programs.singularity = { + enable = true; + inherit package; + }; + }).config.programs.singularity.packageOverriden.overrideAttrs (oldAttrs: { + meta = oldAttrs.meta // { + description = ""; + longDescription = '' + This package produces identical store derivations to `pkgs.${packageName}` + overriden and installed by the NixOS module `programs.singularity` + with default configuration. + + This is for binary substitutes only. Use pkgs.${packageName} instead. + ''; + }; + }); +in +{ + inherit apptainer singularity; + + apptainer-overriden-nixos = genOverridenNixos apptainer "apptainer"; + singularity-overriden-nixos = genOverridenNixos singularity "singularity"; +} |