diff options
Diffstat (limited to 'nixpkgs/pkgs/build-support/kernel/make-initrd-ng')
5 files changed, 476 insertions, 0 deletions
diff --git a/nixpkgs/pkgs/build-support/kernel/make-initrd-ng/Cargo.lock b/nixpkgs/pkgs/build-support/kernel/make-initrd-ng/Cargo.lock new file mode 100644 index 000000000000..61f71f642777 --- /dev/null +++ b/nixpkgs/pkgs/build-support/kernel/make-initrd-ng/Cargo.lock @@ -0,0 +1,111 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "eyre" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "goblin" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7666983ed0dd8d21a6f6576ee00053ca0926fb281a5522577a4dbd0f1b54143" +dependencies = [ + "log", + "plain", + "scroll", +] + +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "make-initrd-ng" +version = "0.1.0" +dependencies = [ + "eyre", + "goblin", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "proc-macro2" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "scroll" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c565b551bafbef4157586fa379538366e4385d42082f255bfd96e4fe8519da" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "2.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" diff --git a/nixpkgs/pkgs/build-support/kernel/make-initrd-ng/Cargo.toml b/nixpkgs/pkgs/build-support/kernel/make-initrd-ng/Cargo.toml new file mode 100644 index 000000000000..028833c12bb5 --- /dev/null +++ b/nixpkgs/pkgs/build-support/kernel/make-initrd-ng/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "make-initrd-ng" +version = "0.1.0" +authors = ["Will Fancher <elvishjerricco@gmail.com>"] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +eyre = "0.6.8" +goblin = "0.5.0" diff --git a/nixpkgs/pkgs/build-support/kernel/make-initrd-ng/README.md b/nixpkgs/pkgs/build-support/kernel/make-initrd-ng/README.md new file mode 100644 index 000000000000..d92b7eab7fe1 --- /dev/null +++ b/nixpkgs/pkgs/build-support/kernel/make-initrd-ng/README.md @@ -0,0 +1,82 @@ +# What is this for? + +NixOS's traditional initrd is generated by listing the paths that +should be included in initrd and copying the full runtime closure of +those paths into the archive. For most things, like almost any +executable, this involves copying the entirety of huge packages like +glibc, when only things like the shared library files are needed. To +solve this, NixOS does a variety of patchwork to edit the files being +copied in so they only refer to small, patched up paths. For instance, +executables and their shared library dependencies are copied into an +`extraUtils` derivation, and every ELF file is patched to refer to +files in that output. + +The problem with this is that it is often difficult to correctly patch +some things. For instance, systemd bakes the path to the `mount` +command into the binary, so patchelf is no help. Instead, it's very +often easier to simply copy the desired files to their original store +locations in initrd and not copy their entire runtime closure. This +does mean that it is the burden of the developer to ensure that all +necessary dependencies are copied in, as closures won't be +consulted. However, it is rare that full closures are actually +desirable, so in the traditional initrd, the developer was likely to +do manual work on patching the dependencies explicitly anyway. + +# How it works + +This program is similar to its inspiration (`find-libs` from the +traditional initrd), except that it also handles symlinks and +directories according to certain rules. As input, it receives a +sequence of pairs of paths. The first path is an object to copy into +initrd. The second path (if not empty) is the path to a symlink that +should be placed in the initrd, pointing to that object. How that +object is copied depends on its type. + +1. A regular file is copied directly to the same absolute path in the + initrd. + + - If it is *also* an ELF file, then all of its direct shared + library dependencies are also listed as objects to be copied. + + - If an unwrapped file exists as `.[filename]-wrapped`, then it is + also listed as an object to be copied. + +2. A directory's direct children are listed as objects to be copied, + and a directory at the same absolute path in the initrd is created. + +3. A symlink's target is listed as an object to be copied. + +There are a couple of quirks to mention here. First, the term "object" +refers to the final file path that the developer intends to have +copied into initrd. This means any parent directory is not considered +an object just because its child was listed as an object in the +program input; instead those intermediate directories are simply +created in support of the target object. Second, shared libraries, +directory children, and symlink targets aren't immediately recursed, +because they simply get listed as objects themselves, and are +therefore traversed when they themselves are processed. Finally, +symlinks in the intermediate directories leading to an object are +preserved, meaning an input object `/a/symlink/b` will just result in +initrd containing `/a/symlink -> /target/b` and `/target/b`, even if +`/target` has other children. Preserving symlinks in this manner is +important for things like systemd. + +These rules automate the most important and obviously necessary +copying that needs to be done in most cases, allowing programs and +configuration files to go unpatched, while keeping the content of the +initrd to a minimum. + +# Why Rust? + +- A prototype of this logic was written in Bash, in an attempt to keep + with its `find-libs` ancestor, but that program was difficult to + write, and ended up taking several minutes to run. This program runs + in less than a second, and the code is substantially easier to work + with. + +- This will not require end users to install a rust toolchain to use + NixOS, as long as this tool is cached by Hydra. And if you're + bootstrapping NixOS from source, rustc is already required anyway. + +- Rust was favored over Python for its type system, and because if you + want to go fast, why not go *really fast*? diff --git a/nixpkgs/pkgs/build-support/kernel/make-initrd-ng/src/main.rs b/nixpkgs/pkgs/build-support/kernel/make-initrd-ng/src/main.rs new file mode 100644 index 000000000000..daa688976c6c --- /dev/null +++ b/nixpkgs/pkgs/build-support/kernel/make-initrd-ng/src/main.rs @@ -0,0 +1,268 @@ +use std::collections::{HashSet, VecDeque}; +use std::env; +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::hash::Hash; +use std::io::{BufRead, BufReader}; +use std::iter::FromIterator; +use std::os::unix; +use std::path::{Component, Path, PathBuf}; +use std::process::Command; + +use eyre::Context; +use goblin::{elf::Elf, Object}; + +struct NonRepeatingQueue<T> { + queue: VecDeque<T>, + seen: HashSet<T>, +} + +impl<T> NonRepeatingQueue<T> { + fn new() -> NonRepeatingQueue<T> { + NonRepeatingQueue { + queue: VecDeque::new(), + seen: HashSet::new(), + } + } +} + +impl<T: Clone + Eq + Hash> NonRepeatingQueue<T> { + fn push_back(&mut self, value: T) -> bool { + if self.seen.contains(&value) { + false + } else { + self.seen.insert(value.clone()); + self.queue.push_back(value); + true + } + } + + fn pop_front(&mut self) -> Option<T> { + self.queue.pop_front() + } +} + +fn add_dependencies<P: AsRef<Path> + AsRef<OsStr>>( + source: P, + elf: Elf, + queue: &mut NonRepeatingQueue<Box<Path>>, +) { + if let Some(interp) = elf.interpreter { + queue.push_back(Box::from(Path::new(interp))); + } + + let rpaths = if elf.runpaths.len() > 0 { + elf.runpaths + } else if elf.rpaths.len() > 0 { + elf.rpaths + } else { + vec![] + }; + + let rpaths_as_path = rpaths + .into_iter() + .flat_map(|p| p.split(":")) + .map(|p| Box::<Path>::from(Path::new(p))) + .collect::<Vec<_>>(); + + for line in elf.libraries { + let mut found = false; + for path in &rpaths_as_path { + let lib = path.join(line); + if lib.exists() { + // No need to recurse. The queue will bring it back round. + queue.push_back(Box::from(lib.as_path())); + found = true; + break; + } + } + if !found { + // glibc makes it tricky to make this an error because + // none of the files have a useful rpath. + println!( + "Warning: Couldn't satisfy dependency {} for {:?}", + line, + OsStr::new(&source) + ); + } + } +} + +fn copy_file< + P: AsRef<Path> + AsRef<OsStr> + std::fmt::Debug, + S: AsRef<Path> + AsRef<OsStr> + std::fmt::Debug, +>( + source: P, + target: S, + queue: &mut NonRepeatingQueue<Box<Path>>, +) -> eyre::Result<()> { + fs::copy(&source, &target) + .wrap_err_with(|| format!("failed to copy {:?} to {:?}", source, target))?; + + let contents = + fs::read(&source).wrap_err_with(|| format!("failed to read from {:?}", source))?; + + if let Ok(Object::Elf(e)) = Object::parse(&contents) { + add_dependencies(source, e, queue); + + // Make file writable to strip it + let mut permissions = fs::metadata(&target) + .wrap_err_with(|| format!("failed to get metadata for {:?}", target))? + .permissions(); + permissions.set_readonly(false); + fs::set_permissions(&target, permissions) + .wrap_err_with(|| format!("failed to set readonly flag to false for {:?}", target))?; + + // Strip further than normal + if let Ok(strip) = env::var("STRIP") { + if !Command::new(strip) + .arg("--strip-all") + .arg(OsStr::new(&target)) + .output()? + .status + .success() + { + println!("{:?} was not successfully stripped.", OsStr::new(&target)); + } + } + }; + + Ok(()) +} + +fn queue_dir<P: AsRef<Path> + std::fmt::Debug>( + source: P, + queue: &mut NonRepeatingQueue<Box<Path>>, +) -> eyre::Result<()> { + for entry in + fs::read_dir(&source).wrap_err_with(|| format!("failed to read dir {:?}", source))? + { + let entry = entry?; + // No need to recurse. The queue will bring us back round here on its own. + queue.push_back(Box::from(entry.path().as_path())); + } + + Ok(()) +} + +fn handle_path( + root: &Path, + p: &Path, + queue: &mut NonRepeatingQueue<Box<Path>>, +) -> eyre::Result<()> { + let mut source = PathBuf::new(); + let mut target = Path::new(root).to_path_buf(); + let mut iter = p.components().peekable(); + while let Some(comp) = iter.next() { + match comp { + Component::Prefix(_) => panic!("This tool is not meant for Windows"), + Component::RootDir => { + target.clear(); + target.push(root); + source.clear(); + source.push("/"); + } + Component::CurDir => {} + Component::ParentDir => { + // Don't over-pop the target if the path has too many ParentDirs + if source.pop() { + target.pop(); + } + } + Component::Normal(name) => { + target.push(name); + source.push(name); + let typ = fs::symlink_metadata(&source) + .wrap_err_with(|| format!("failed to get symlink metadata for {:?}", source))? + .file_type(); + if typ.is_file() && !target.exists() { + copy_file(&source, &target, queue)?; + + if let Some(filename) = source.file_name() { + source.set_file_name(OsString::from_iter([ + OsStr::new("."), + filename, + OsStr::new("-wrapped"), + ])); + + let wrapped_path = source.as_path(); + if wrapped_path.exists() { + queue.push_back(Box::from(wrapped_path)); + } + } + } else if typ.is_symlink() { + let link_target = fs::read_link(&source) + .wrap_err_with(|| format!("failed to resolve symlink of {:?}", source))?; + + // Create the link, then push its target to the queue + if !target.exists() && !target.is_symlink() { + unix::fs::symlink(&link_target, &target).wrap_err_with(|| { + format!("failed to symlink {:?} to {:?}", link_target, target) + })?; + } + source.pop(); + source.push(link_target); + while let Some(c) = iter.next() { + source.push(c); + } + let link_target_path = source.as_path(); + if link_target_path.exists() { + queue.push_back(Box::from(link_target_path)); + } + break; + } else if typ.is_dir() { + if !target.exists() { + fs::create_dir(&target) + .wrap_err_with(|| format!("failed to create dir {:?}", target))?; + } + + // Only recursively copy if the directory is the target object + if iter.peek().is_none() { + queue_dir(&source, queue) + .wrap_err_with(|| format!("failed to queue dir {:?}", source))?; + } + } + } + } + } + + Ok(()) +} + +fn main() -> eyre::Result<()> { + let args: Vec<String> = env::args().collect(); + let input = + fs::File::open(&args[1]).wrap_err_with(|| format!("failed to open file {:?}", &args[1]))?; + let output = &args[2]; + let out_path = Path::new(output); + + let mut queue = NonRepeatingQueue::<Box<Path>>::new(); + + let mut lines = BufReader::new(input).lines(); + while let Some(obj) = lines.next() { + // Lines should always come in pairs + let obj = obj?; + let sym = lines.next().unwrap()?; + + let obj_path = Path::new(&obj); + queue.push_back(Box::from(obj_path)); + if !sym.is_empty() { + println!("{} -> {}", &sym, &obj); + // We don't care about preserving symlink structure here + // nearly as much as for the actual objects. + let link_string = format!("{}/{}", output, sym); + let link_path = Path::new(&link_string); + let mut link_parent = link_path.to_path_buf(); + link_parent.pop(); + fs::create_dir_all(&link_parent) + .wrap_err_with(|| format!("failed to create directories to {:?}", link_parent))?; + unix::fs::symlink(obj_path, link_path) + .wrap_err_with(|| format!("failed to symlink {:?} to {:?}", obj_path, link_path))?; + } + } + while let Some(obj) = queue.pop_front() { + handle_path(out_path, &*obj, &mut queue)?; + } + + Ok(()) +} diff --git a/nixpkgs/pkgs/build-support/kernel/make-initrd-ng/update.sh b/nixpkgs/pkgs/build-support/kernel/make-initrd-ng/update.sh new file mode 100755 index 000000000000..ffc5ad3917f7 --- /dev/null +++ b/nixpkgs/pkgs/build-support/kernel/make-initrd-ng/update.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env nix-shell +#!nix-shell -p cargo -i bash +cd "$(dirname "$0")" +cargo update |