Merge pull request #275539 from tweag/by-name-enforce

[RFC 140 2a] `pkgs/by-name`: Enforce for new packages

76 files changed, 386 insertions, 85 deletions

diff --git a/pkgs/test/nixpkgs-check-by-name/README.md b/pkgs/test/nixpkgs-check-by-name/README.md
index 871847bd74cc..d779529c7baf 100644
--- a/pkgs/test/nixpkgs-check-by-name/README.md
+++ b/pkgs/test/nixpkgs-check-by-name/README.md
@@ -45,6 +45,8 @@ The current ratchets are:
 
 - New manual definitions of `pkgs.${name}` (e.g. in `pkgs/top-level/all-packages.nix`) with `args = { }`
   (see [nix evaluation checks](#nix-evaluation-checks)) must not be introduced.
+- New top-level packages defined using `pkgs.callPackage` must be defined with a package directory.
+  - Once a top-level package uses `pkgs/by-name`, it also can't be moved back out of it.
 
 ## Development
 
diff --git a/pkgs/test/nixpkgs-check-by-name/default.nix b/pkgs/test/nixpkgs-check-by-name/default.nix
index fc24b1fd3398..d2de2d960042 100644
--- a/pkgs/test/nixpkgs-check-by-name/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/default.nix
@@ -9,6 +9,7 @@
 }:
 let
   runtimeExprPath = ./src/eval.nix;
+  nixpkgsLibPath = ../../../lib;
   package =
     rustPlatform.buildRustPackage {
       name = "nixpkgs-check-by-name";
@@ -30,6 +31,8 @@ let
         export NIX_STATE_DIR=$TEST_ROOT/var/nix
         export NIX_STORE_DIR=$TEST_ROOT/store
 
+        export NIXPKGS_LIB_PATH=${nixpkgsLibPath}
+
         # Ensure that even if tests run in parallel, we don't get an error
         # We'd run into https://github.com/NixOS/nix/issues/2706 unless the store is initialised first
         nix-store --init
@@ -44,6 +47,7 @@ let
       '';
       passthru.shell = mkShell {
         env.NIX_CHECK_BY_NAME_EXPR_PATH = toString runtimeExprPath;
+        env.NIXPKGS_LIB_PATH = toString nixpkgsLibPath;
         inputsFrom = [ package ];
       };
     };
diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.nix b/pkgs/test/nixpkgs-check-by-name/src/eval.nix
index 7707dc732b70..87c54b6444ee 100644
--- a/pkgs/test/nixpkgs-check-by-name/src/eval.nix
+++ b/pkgs/test/nixpkgs-check-by-name/src/eval.nix
@@ -79,15 +79,37 @@ let
         };
       };
 
-  attrInfos = map (name: [
-    name
-    (
+  byNameAttrs = builtins.listToAttrs (map (name: {
+    inherit name;
+    value.ByName =
       if ! pkgs ? ${name} then
         { Missing = null; }
       else
-        { Existing = attrInfo name pkgs.${name}; }
-    )
-  ]) attrs;
+        { Existing = attrInfo name pkgs.${name}; };
+  }) attrs);
 
+  # Information on all attributes that exist but are not in pkgs/by-name.
+  # We need this to enforce pkgs/by-name for new packages
+  nonByNameAttrs = builtins.mapAttrs (name: value:
+    let
+      output = attrInfo name value;
+      result = builtins.tryEval (builtins.deepSeq output null);
+    in
+    {
+      NonByName =
+        if result.success then
+          { EvalSuccess = output; }
+        else
+          { EvalFailure = null; };
+    }
+  ) (builtins.removeAttrs pkgs attrs);
+
+  # All attributes
+  attributes = byNameAttrs // nonByNameAttrs;
 in
-attrInfos
+# We output them in the form [ [ <name> <value> ] ]` such that the Rust side
+# doesn't need to sort them again to get deterministic behavior (good for testing)
+map (name: [
+  name
+  attributes.${name}
+]) (builtins.attrNames attributes)
diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.rs b/pkgs/test/nixpkgs-check-by-name/src/eval.rs
index 65f71ccafc6f..b411a2a3c601 100644
--- a/pkgs/test/nixpkgs-check-by-name/src/eval.rs
+++ b/pkgs/test/nixpkgs-check-by-name/src/eval.rs
@@ -2,6 +2,8 @@ use crate::nixpkgs_problem::NixpkgsProblem;
 use crate::ratchet;
 use crate::structure;
 use crate::validation::{self, Validation::Success};
+use std::collections::HashMap;
+use std::ffi::OsString;
 use std::path::Path;
 
 use anyhow::Context;
@@ -12,6 +14,21 @@ use tempfile::NamedTempFile;
 
 /// Attribute set of this structure is returned by eval.nix
 #[derive(Deserialize)]
+enum Attribute {
+    /// An attribute that should be defined via pkgs/by-name
+    ByName(ByNameAttribute),
+    /// An attribute not defined via pkgs/by-name
+    NonByName(NonByNameAttribute),
+}
+
+#[derive(Deserialize)]
+enum NonByNameAttribute {
+    /// The attribute doesn't evaluate
+    EvalFailure,
+    EvalSuccess(AttributeInfo),
+}
+
+#[derive(Deserialize)]
 enum ByNameAttribute {
     /// The attribute doesn't exist at all
     Missing,
@@ -56,7 +73,7 @@ enum CallPackageVariant {
 pub fn check_values(
     nixpkgs_path: &Path,
     package_names: Vec<String>,
-    eval_accessible_paths: &[&Path],
+    eval_nix_path: &HashMap<String, PathBuf>,
 ) -> validation::Result<ratchet::Nixpkgs> {
     // Write the list of packages we need to check into a temporary JSON file.
     // This can then get read by the Nix evaluation.
@@ -105,9 +122,13 @@ pub fn check_values(
         .arg(nixpkgs_path);
 
     // Also add extra paths that need to be accessible
-    for path in eval_accessible_paths {
+    for (name, path) in eval_nix_path {
         command.arg("-I");
-        command.arg(path);
+        let mut name_value = OsString::new();
+        name_value.push(name);
+        name_value.push("=");
+        name_value.push(path);
+        command.arg(name_value);
     }
     command.args(["-I", &expr_path]);
     command.arg(expr_path);
@@ -120,7 +141,7 @@ pub fn check_values(
         anyhow::bail!("Failed to run command {command:?}");
     }
     // Parse the resulting JSON value
-    let attributes: Vec<(String, ByNameAttribute)> = serde_json::from_slice(&result.stdout)
+    let attributes: Vec<(String, Attribute)> = serde_json::from_slice(&result.stdout)
         .with_context(|| {
             format!(
                 "Failed to deserialise {}",
@@ -133,30 +154,86 @@ pub fn check_values(
             let relative_package_file = structure::relative_file_for_package(&attribute_name);
 
             use ratchet::RatchetState::*;
+            use Attribute::*;
             use AttributeInfo::*;
             use ByNameAttribute::*;
             use CallPackageVariant::*;
+            use NonByNameAttribute::*;
 
             let check_result = match attribute_value {
-                Missing => NixpkgsProblem::UndefinedAttr {
+                // The attribute succeeds evaluation and is NOT defined in pkgs/by-name
+                NonByName(EvalSuccess(attribute_info)) => {
+                    let uses_by_name = match attribute_info {
+                        // In these cases the package doesn't qualify for being in pkgs/by-name,
+                        // so the UsesByName ratchet is already as tight as it can be
+                        NonAttributeSet => Success(Tight),
+                        NonCallPackage => Success(Tight),
+                        // This is an odd case when _internalCallByNamePackageFile is used to define a package.
+                        CallPackage(CallPackageInfo {
+                            call_package_variant: Auto,
+                            ..
+                        }) => NixpkgsProblem::InternalCallPackageUsed {
+                            attr_name: attribute_name.clone(),
+                        }
+                        .into(),
+                        // Only derivations can be in pkgs/by-name,
+                        // so this attribute doesn't qualify
+                        CallPackage(CallPackageInfo {
+                            is_derivation: false,
+                            ..
+                        }) => Success(Tight),
+
+                        // The case of an attribute that qualifies:
+                        // - Uses callPackage
+                        // - Is a derivation
+                        CallPackage(CallPackageInfo {
+                            is_derivation: true,
+                            call_package_variant: Manual { path, empty_arg },
+                        }) => Success(Loose(ratchet::UsesByName {
+                            call_package_path: path,
+                            empty_arg,
+                        })),
+                    };
+                    uses_by_name.map(|x| ratchet::Package {
+                        empty_non_auto_called: Tight,
+                        uses_by_name: x,
+                    })
+                }
+                NonByName(EvalFailure) => {
+                    // This is a bit of an odd case: We don't even _know_ whether this attribute
+                    // would qualify for using pkgs/by-name. We can either:
+                    // - Assume it's not using pkgs/by-name, which has the problem that if a
+                    //   package evaluation gets broken temporarily, the fix can remove it from
+                    //   pkgs/by-name again
+                    // - Assume it's using pkgs/by-name already, which has the problem that if a
+                    //   package evaluation gets broken temporarily, fixing it requires a move to
+                    //   pkgs/by-name
+                    // We choose the latter, since we want to move towards pkgs/by-name, not away
+                    // from it
+                    Success(ratchet::Package {
+                        empty_non_auto_called: Tight,
+                        uses_by_name: Tight,
+                    })
+                }
+                ByName(Missing) => NixpkgsProblem::UndefinedAttr {
                     relative_package_file: relative_package_file.clone(),
                     package_name: attribute_name.clone(),
                 }
                 .into(),
-                Existing(NonAttributeSet) => NixpkgsProblem::NonDerivation {
+                ByName(Existing(NonAttributeSet)) => NixpkgsProblem::NonDerivation {
                     relative_package_file: relative_package_file.clone(),
                     package_name: attribute_name.clone(),
                 }
                 .into(),
-                Existing(NonCallPackage) => NixpkgsProblem::WrongCallPackage {
+                ByName(Existing(NonCallPackage)) => NixpkgsProblem::WrongCallPackage {
                     relative_package_file: relative_package_file.clone(),
                     package_name: attribute_name.clone(),
                 }
                 .into(),
-                Existing(CallPackage(CallPackageInfo {
+                ByName(Existing(CallPackage(CallPackageInfo {
                     is_derivation,
                     call_package_variant,
-                })) => {
+                }))) => {
                     let check_result = if !is_derivation {
                         NixpkgsProblem::NonDerivation {
                             relative_package_file: relative_package_file.clone(),
@@ -170,6 +247,7 @@ pub fn check_values(
                     check_result.and(match &call_package_variant {
                         Auto => Success(ratchet::Package {
                             empty_non_auto_called: Tight,
+                            uses_by_name: Tight,
                         }),
                         Manual { path, empty_arg } => {
                             let correct_file = if let Some(call_package_path) = path {
@@ -186,6 +264,7 @@ pub fn check_values(
                                     } else {
                                         Tight
                                     },
+                                    uses_by_name: Tight,
                                 })
                             } else {
                                 NixpkgsProblem::WrongCallPackage {
@@ -203,7 +282,7 @@ pub fn check_values(
     ));
 
     Ok(check_result.map(|elems| ratchet::Nixpkgs {
-        package_names,
+        package_names: elems.iter().map(|(name, _)| name.to_owned()).collect(),
         package_map: elems.into_iter().collect(),
     }))
 }
diff --git a/pkgs/test/nixpkgs-check-by-name/src/main.rs b/pkgs/test/nixpkgs-check-by-name/src/main.rs
index d7627acb5fee..273ebca1643e 100644
--- a/pkgs/test/nixpkgs-check-by-name/src/main.rs
+++ b/pkgs/test/nixpkgs-check-by-name/src/main.rs
@@ -12,6 +12,7 @@ use crate::validation::Validation::Success;
 use anyhow::Context;
 use clap::Parser;
 use colored::Colorize;
+use std::collections::HashMap;
 use std::io;
 use std::path::{Path, PathBuf};
 use std::process::ExitCode;
@@ -44,7 +45,12 @@ pub struct Args {
 
 fn main() -> ExitCode {
     let args = Args::parse();
-    match process(&args.base, &args.nixpkgs, &[], &mut io::stderr()) {
+    match process(
+        &args.base,
+        &args.nixpkgs,
+        &HashMap::new(),
+        &mut io::stderr(),
+    ) {
         Ok(true) => {
             eprintln!("{}", "Validated successfully".green());
             ExitCode::SUCCESS
@@ -77,15 +83,15 @@ fn main() -> ExitCode {
 pub fn process<W: io::Write>(
     base_nixpkgs: &Path,
     main_nixpkgs: &Path,
-    eval_accessible_paths: &[&Path],
+    eval_nix_path: &HashMap<String, PathBuf>,
     error_writer: &mut W,
 ) -> anyhow::Result<bool> {
     // Check the main Nixpkgs first
-    let main_result = check_nixpkgs(main_nixpkgs, eval_accessible_paths, error_writer)?;
+    let main_result = check_nixpkgs(main_nixpkgs, eval_nix_path, error_writer)?;
     let check_result = main_result.result_map(|nixpkgs_version| {
         // If the main Nixpkgs doesn't have any problems, run the ratchet checks against the base
         // Nixpkgs
-        check_nixpkgs(base_nixpkgs, eval_accessible_paths, error_writer)?.result_map(
+        check_nixpkgs(base_nixpkgs, eval_nix_path, error_writer)?.result_map(
             |base_nixpkgs_version| {
                 Ok(ratchet::Nixpkgs::compare(
                     base_nixpkgs_version,
@@ -113,7 +119,7 @@ pub fn process<W: io::Write>(
 /// ratchet check against another result.
 pub fn check_nixpkgs<W: io::Write>(
     nixpkgs_path: &Path,
-    eval_accessible_paths: &[&Path],
+    eval_nix_path: &HashMap<String, PathBuf>,
     error_writer: &mut W,
 ) -> validation::Result<ratchet::Nixpkgs> {
     Ok({
@@ -134,7 +140,7 @@ pub fn check_nixpkgs<W: io::Write>(
         } else {
             check_structure(&nixpkgs_path)?.result_map(|package_names|
                 // Only if we could successfully parse the structure, we do the evaluation checks
-                eval::check_values(&nixpkgs_path, package_names, eval_accessible_paths))?
+                eval::check_values(&nixpkgs_path, package_names, eval_nix_path))?
         }
     })
 }
@@ -144,8 +150,10 @@ mod tests {
     use crate::process;
     use crate::utils;
     use anyhow::Context;
+    use std::collections::HashMap;
     use std::fs;
     use std::path::Path;
+    use std::path::PathBuf;
     use tempfile::{tempdir_in, TempDir};
 
     #[test]
@@ -226,7 +234,19 @@ mod tests {
     }
 
     fn test_nixpkgs(name: &str, path: &Path, expected_errors: &str) -> anyhow::Result<()> {
-        let extra_nix_path = Path::new("tests/mock-nixpkgs.nix");
+        let eval_nix_path = HashMap::from([
+            (
+                "test-nixpkgs".to_string(),
+                PathBuf::from("tests/mock-nixpkgs.nix"),
+            ),
+            (
+                "test-nixpkgs/lib".to_string(),
+                PathBuf::from(
+                    std::env::var("NIXPKGS_LIB_PATH")
+                        .with_context(|| "Could not get environment variable NIXPKGS_LIB_PATH")?,
+                ),
+            ),
+        ]);
 
         let base_path = path.join("base");
         let base_nixpkgs = if base_path.exists() {
@@ -238,7 +258,7 @@ mod tests {
         // We don't want coloring to mess up the tests
         let writer = temp_env::with_var("NO_COLOR", Some("1"), || -> anyhow::Result<_> {
             let mut writer = vec![];
-            process(base_nixpkgs, &path, &[&extra_nix_path], &mut writer)
+            process(base_nixpkgs, &path, &eval_nix_path, &mut writer)
                 .with_context(|| format!("Failed test case {name}"))?;
             Ok(writer)
         })?;
diff --git a/pkgs/test/nixpkgs-check-by-name/src/nixpkgs_problem.rs b/pkgs/test/nixpkgs-check-by-name/src/nixpkgs_problem.rs
index 2344a8cc1325..127583078074 100644
--- a/pkgs/test/nixpkgs-check-by-name/src/nixpkgs_problem.rs
+++ b/pkgs/test/nixpkgs-check-by-name/src/nixpkgs_problem.rs
@@ -1,3 +1,4 @@
+use crate::structure;
 use crate::utils::PACKAGE_NIX_FILENAME;
 use rnix::parser::ParseError;
 use std::ffi::OsString;
@@ -87,6 +88,19 @@ pub enum NixpkgsProblem {
         text: String,
         io_error: io::Error,
     },
+    InternalCallPackageUsed {
+        attr_name: String,
+    },
+    MovedOutOfByName {
+        package_name: String,
+        call_package_path: Option<PathBuf>,
+        empty_arg: bool,
+    },
+    NewPackageNotUsingByName {
+        package_name: String,
+        call_package_path: Option<PathBuf>,
+        empty_arg: bool,
+    },
 }
 
 impl fmt::Display for NixpkgsProblem {
@@ -213,6 +227,53 @@ impl fmt::Display for NixpkgsProblem {
                     subpath.display(),
                     text,
                 ),
+            NixpkgsProblem::InternalCallPackageUsed { attr_name } =>
+                write!(
+                    f,
+                    "pkgs.{attr_name}: This attribute is defined using `_internalCallByNamePackageFile`, which is an internal function not intended for manual use.",
+                ),
+            NixpkgsProblem::MovedOutOfByName { package_name, call_package_path, empty_arg } => {
+                let call_package_arg =
+                    if let Some(path) = &call_package_path {
+                        format!("./{}", path.display())
+                    } else {
+                        "...".into()
+                    };
+                if *empty_arg {
+                    write!(
+                        f,
+                        "pkgs.{package_name}: This top-level package was previously defined in {}, but is now manually defined as `callPackage {call_package_arg} {{ }}` (e.g. in `pkgs/top-level/all-packages.nix`). Please move the package back and remove the manual `callPackage`.",
+                        structure::relative_file_for_package(package_name).display(),
+                        )
+                } else {
+                    // This can happen if users mistakenly assume that for custom arguments,
+                    // pkgs/by-name can't be used.
+                    write!(
+                        f,
+                        "pkgs.{package_name}: This top-level package was previously defined in {}, but is now manually defined as `callPackage {call_package_arg} {{ ... }}` (e.g. in `pkgs/top-level/all-packages.nix`). While the manual `callPackage` is still needed, it's not necessary to move the package files.",
+                        structure::relative_file_for_package(package_name).display(),
+                        )
+                }
+            },
+            NixpkgsProblem::NewPackageNotUsingByName { package_name, call_package_path, empty_arg } => {
+                let call_package_arg =
+                    if let Some(path) = &call_package_path {
+                        format!("./{}", path.display())
+                    } else {
+                        "...".into()
+                    };
+                let extra =
+                    if *empty_arg {
+                        "Since the second `callPackage` argument is `{ }`, no manual `callPackage` (e.g. in `pkgs/top-level/all-packages.nix`) is needed anymore."
+                    } else {
+                        "Since the second `callPackage` argument is not `{ }`, the manual `callPackage` (e.g. in `pkgs/top-level/all-packages.nix`) is still needed."
+                    };
+                write!(
+                    f,
+                    "pkgs.{package_name}: This is a new top-level package of the form `callPackage {call_package_arg} {{ }}`. Please define it in {} instead. See `pkgs/by-name/README.md` for more details. {extra}",
+                    structure::relative_file_for_package(package_name).display(),
+                )
+            },
         }
     }
 }
diff --git a/pkgs/test/nixpkgs-check-by-name/src/ratchet.rs b/pkgs/test/nixpkgs-check-by-name/src/ratchet.rs
index 85feb0eee62f..f8c129626cc0 100644
--- a/pkgs/test/nixpkgs-check-by-name/src/ratchet.rs
+++ b/pkgs/test/nixpkgs-check-by-name/src/ratchet.rs
@@ -6,11 +6,12 @@ use crate::nixpkgs_problem::NixpkgsProblem;
 use crate::structure;
 use crate::validation::{self, Validation, Validation::Success};
 use std::collections::HashMap;
+use std::path::PathBuf;
 
 /// The ratchet value for the entirety of Nixpkgs.
 #[derive(Default)]
 pub struct Nixpkgs {
-    /// Sorted list of attributes in package_map
+    /// Sorted list of packages in package_map
     pub package_names: Vec<String>,
     /// The ratchet values for all packages
     pub package_map: HashMap<String, Package>,
@@ -29,20 +30,30 @@ impl Nixpkgs {
     }
 }
 
-/// The ratchet value for a single package in `pkgs/by-name`
+/// The ratchet value for a top-level package
 pub struct Package {
     /// The ratchet value for the check for non-auto-called empty arguments
     pub empty_non_auto_called: RatchetState<EmptyNonAutoCalled>,
+
+    /// The ratchet value for the check for new packages using pkgs/by-name
+    pub uses_by_name: RatchetState<UsesByName>,
 }
 
 impl Package {
-    /// Validates the ratchet checks for a single package defined in `pkgs/by-name`
+    /// Validates the ratchet checks for a top-level package
     pub fn compare(name: &str, optional_from: Option<&Self>, to: &Self) -> Validation<()> {
-        RatchetState::<EmptyNonAutoCalled>::compare(
-            name,
-            optional_from.map(|x| &x.empty_non_auto_called),
-            &to.empty_non_auto_called,
-        )
+        validation::sequence_([
+            RatchetState::<EmptyNonAutoCalled>::compare(
+                name,
+                optional_from.map(|x| &x.empty_non_auto_called),
+                &to.empty_non_auto_called,
+            ),
+            RatchetState::<UsesByName>::compare(
+                name,
+                optional_from.map(|x| &x.uses_by_name),
+                &to.uses_by_name,
+            ),
+        ])
     }
 }
 
@@ -102,3 +113,34 @@ impl ToNixpkgsProblem for EmptyNonAutoCalled {
         }
     }
 }
+
+/// The ratchet value of an attribute
+/// for the check that new packages use pkgs/by-name
+///
+/// This checks that all new package defined using callPackage must be defined via pkgs/by-name
+/// It also checks that once a package uses pkgs/by-name, it can't switch back to all-packages.nix
+#[derive(Clone)]
+pub struct UsesByName {
+    /// The first callPackage argument, used for better errors
+    pub call_package_path: Option<PathBuf>,
+    /// Whether the second callPackage argument is empty, used for better errors
+    pub empty_arg: bool,
+}
+
+impl ToNixpkgsProblem for UsesByName {
+    fn to_nixpkgs_problem(name: &str, a: &Self, existed_before: bool) -> NixpkgsProblem {
+        if existed_before {
+            NixpkgsProblem::MovedOutOfByName {
+                package_name: name.to_owned(),
+                call_package_path: a.call_package_path.clone(),
+                empty_arg: a.empty_arg,
+            }
+        } else {
+            NixpkgsProblem::NewPackageNotUsingByName {
+                package_name: name.to_owned(),
+                call_package_path: a.call_package_path.clone(),
+                empty_arg: a.empty_arg,
+            }
+        }
+    }
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/default.nix
index 793dfabd6553..bd4825f8bad8 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/broken-autocall/default.nix
@@ -1,4 +1,4 @@
 args:
 builtins.removeAttrs
-  (import ../mock-nixpkgs.nix { root = ./.; } args)
+  (import <test-nixpkgs> { root = ./.; } args)
   [ "foo" ]
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/empty-base/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/empty-base/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/empty-base/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/empty-base/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/incorrect-shard/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/all-packages.nix
new file mode 100644
index 000000000000..95478a87fb32
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/all-packages.nix
@@ -0,0 +1,3 @@
+self: super: {
+  foo = self._internalCallByNamePackageFile ./foo.nix;
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/expected b/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/expected
new file mode 100644
index 000000000000..404795ee5c79
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/expected
@@ -0,0 +1 @@
+pkgs.foo: This attribute is defined using `_internalCallByNamePackageFile`, which is an internal function not intended for manual use.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/foo.nix b/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/foo.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/foo.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/pkgs/by-name/README.md b/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/pkgs/by-name/README.md
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/pkgs/by-name/README.md
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/invalid-package-name/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/invalid-shard-name/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/missing-package-nix/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/mock-nixpkgs.nix b/pkgs/test/nixpkgs-check-by-name/tests/mock-nixpkgs.nix
index cb8066062cc6..183f8ff2ae8d 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/mock-nixpkgs.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/mock-nixpkgs.nix
@@ -25,30 +25,14 @@ It returns a Nixpkgs-like function that can be auto-called and evaluates to an a
 let
 
   # Simplified versions of lib functions
-  lib = {
-    fix = f: let x = f x; in x;
-
-    extends = overlay: f: final:
-      let
-        prev = f final;
-      in
-      prev // overlay final prev;
-
-    callPackageWith = autoArgs: fn: args:
-      let
-        f = if builtins.isFunction fn then fn else import fn;
-        fargs = builtins.functionArgs f;
-        allArgs = builtins.intersectAttrs fargs autoArgs // args;
-      in
-      f allArgs;
-
-    isDerivation = value: value.type or null == "derivation";
-  };
+  lib = import <test-nixpkgs/lib>;
 
   # The base fixed-point function to populate the resulting attribute set
   pkgsFun = self: {
     inherit lib;
-    callPackage = lib.callPackageWith self;
+    newScope = extra: lib.callPackageWith (self // extra);
+    callPackage = self.newScope { };
+    callPackages = lib.callPackagesWith self;
     someDrv = { type = "derivation"; };
   };
 
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/all-packages.nix
new file mode 100644
index 000000000000..16834c4f7856
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/all-packages.nix
@@ -0,0 +1,10 @@
+self: super: {
+  foo1 = self.callPackage ({ someDrv }: someDrv) { };
+  foo2 = self.callPackage ./without-config.nix { };
+  foo3 = self.callPackage ({ someDrv, enableFoo }: someDrv) {
+    enableFoo = null;
+  };
+  foo4 = self.callPackage ./with-config.nix {
+    enableFoo = null;
+  };
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/base/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/base/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/base/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/base/pkgs/by-name/fo/foo1/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/base/pkgs/by-name/fo/foo1/package.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/base/pkgs/by-name/fo/foo1/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/base/pkgs/by-name/fo/foo2/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/base/pkgs/by-name/fo/foo2/package.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/base/pkgs/by-name/fo/foo2/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/base/pkgs/by-name/fo/foo3/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/base/pkgs/by-name/fo/foo3/package.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/base/pkgs/by-name/fo/foo3/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/base/pkgs/by-name/fo/foo4/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/base/pkgs/by-name/fo/foo4/package.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/base/pkgs/by-name/fo/foo4/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/expected b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/expected
new file mode 100644
index 000000000000..96da50b52491
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/expected
@@ -0,0 +1,4 @@
+pkgs.foo1: This top-level package was previously defined in pkgs/by-name/fo/foo1/package.nix, but is now manually defined as `callPackage ... { }` (e.g. in `pkgs/top-level/all-packages.nix`). Please move the package back and remove the manual `callPackage`.
+pkgs.foo2: This top-level package was previously defined in pkgs/by-name/fo/foo2/package.nix, but is now manually defined as `callPackage ./without-config.nix { }` (e.g. in `pkgs/top-level/all-packages.nix`). Please move the package back and remove the manual `callPackage`.
+pkgs.foo3: This top-level package was previously defined in pkgs/by-name/fo/foo3/package.nix, but is now manually defined as `callPackage ... { ... }` (e.g. in `pkgs/top-level/all-packages.nix`). While the manual `callPackage` is still needed, it's not necessary to move the package files.
+pkgs.foo4: This top-level package was previously defined in pkgs/by-name/fo/foo4/package.nix, but is now manually defined as `callPackage ./with-config.nix { ... }` (e.g. in `pkgs/top-level/all-packages.nix`). While the manual `callPackage` is still needed, it's not necessary to move the package files.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/pkgs/by-name/README.md b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/pkgs/by-name/README.md
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/pkgs/by-name/README.md
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/with-config.nix b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/with-config.nix
new file mode 100644
index 000000000000..7cca53882ea5
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/with-config.nix
@@ -0,0 +1 @@
+{ someDrv, enableFoo }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/without-config.nix b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/without-config.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/move-to-non-by-name/without-config.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/all-packages.nix
new file mode 100644
index 000000000000..069119ad4c7b
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/all-packages.nix
@@ -0,0 +1,11 @@
+self: super: {
+  before = self.callPackage ({ someDrv }: someDrv) { };
+  new1 = self.callPackage ({ someDrv }: someDrv) { };
+  new2 = self.callPackage ./without-config.nix { };
+  new3 = self.callPackage ({ someDrv, enableNew }: someDrv) {
+    enableNew = null;
+  };
+  new4 = self.callPackage ./with-config.nix {
+    enableNew = null;
+  };
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/base/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/base/all-packages.nix
new file mode 100644
index 000000000000..c2665d04d5f2
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/base/all-packages.nix
@@ -0,0 +1,5 @@
+self: super: {
+
+  before = self.callPackage ({ someDrv }: someDrv) { };
+
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/base/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/base/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/base/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/base/pkgs/by-name/README.md b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/base/pkgs/by-name/README.md
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/base/pkgs/by-name/README.md
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/expected b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/expected
new file mode 100644
index 000000000000..3f294f26dfd8
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/expected
@@ -0,0 +1,4 @@
+pkgs.new1: This is a new top-level package of the form `callPackage ... { }`. Please define it in pkgs/by-name/ne/new1/package.nix instead. See `pkgs/by-name/README.md` for more details. Since the second `callPackage` argument is `{ }`, no manual `callPackage` (e.g. in `pkgs/top-level/all-packages.nix`) is needed anymore.
+pkgs.new2: This is a new top-level package of the form `callPackage ./without-config.nix { }`. Please define it in pkgs/by-name/ne/new2/package.nix instead. See `pkgs/by-name/README.md` for more details. Since the second `callPackage` argument is `{ }`, no manual `callPackage` (e.g. in `pkgs/top-level/all-packages.nix`) is needed anymore.
+pkgs.new3: This is a new top-level package of the form `callPackage ... { }`. Please define it in pkgs/by-name/ne/new3/package.nix instead. See `pkgs/by-name/README.md` for more details. Since the second `callPackage` argument is not `{ }`, the manual `callPackage` (e.g. in `pkgs/top-level/all-packages.nix`) is still needed.
+pkgs.new4: This is a new top-level package of the form `callPackage ./with-config.nix { }`. Please define it in pkgs/by-name/ne/new4/package.nix instead. See `pkgs/by-name/README.md` for more details. Since the second `callPackage` argument is not `{ }`, the manual `callPackage` (e.g. in `pkgs/top-level/all-packages.nix`) is still needed.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/pkgs/by-name/README.md b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/pkgs/by-name/README.md
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/pkgs/by-name/README.md
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/with-config.nix b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/with-config.nix
new file mode 100644
index 000000000000..65bcbf9928a2
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/with-config.nix
@@ -0,0 +1 @@
+{ someDrv, enableNew }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/without-config.nix b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/without-config.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/new-package-non-by-name/without-config.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/no-by-name/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/no-by-name/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/no-by-name/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/no-by-name/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/no-eval/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/no-eval/all-packages.nix
new file mode 100644
index 000000000000..e2831c2d542e
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/no-eval/all-packages.nix
@@ -0,0 +1,3 @@
+self: super: {
+  iDontEval = throw "I don't eval";
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/no-eval/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/no-eval/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/no-eval/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/no-eval/pkgs/by-name/fo/foo/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/no-eval/pkgs/by-name/fo/foo/package.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/no-eval/pkgs/by-name/fo/foo/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/non-attrs/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/non-attrs/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/non-attrs/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/non-attrs/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/non-derivation/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/non-derivation/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/non-derivation/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/non-derivation/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/one-letter/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/one-letter/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/one-letter/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/one-letter/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/only-callPackage-derivations/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/only-callPackage-derivations/all-packages.nix
new file mode 100644
index 000000000000..5b1ed9d2ccda
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/only-callPackage-derivations/all-packages.nix
@@ -0,0 +1,16 @@
+self: super: {
+  alternateCallPackage = self.myScope.callPackage ({ myScopeValue, someDrv }:
+    assert myScopeValue;
+    someDrv
+  ) { };
+
+  myScope = self.lib.makeScope self.newScope (self: {
+    myScopeValue = true;
+  });
+
+  myPackages = self.callPackages ({ someDrv }: {
+    a = someDrv;
+    b = someDrv;
+  }) { };
+  inherit (self.myPackages) a b;
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/only-callPackage-derivations/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/only-callPackage-derivations/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/only-callPackage-derivations/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/only-callPackage-derivations/pkgs/by-name/README.md b/pkgs/test/nixpkgs-check-by-name/tests/only-callPackage-derivations/pkgs/by-name/README.md
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/only-callPackage-derivations/pkgs/by-name/README.md
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-different-file/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/default.nix
index 2875ea6327ef..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/default.nix
@@ -1 +1 @@
-import ../../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/default.nix
index 2875ea6327ef..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/default.nix
@@ -1 +1 @@
-import ../../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-success/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-success/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/override-success/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/override-success/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/package-dir-is-file/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/package-nix-dir/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-absolute/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-escape/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-escape/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/ref-escape/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-escape/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-nix-path/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-parse-failure/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-path-subexpr/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/ref-success/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/ref-success/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/ref-success/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/ref-success/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/shard-file/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/shard-file/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/shard-file/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/shard-file/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/all-packages.nix
new file mode 100644
index 000000000000..688f52b9358f
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/all-packages.nix
@@ -0,0 +1,6 @@
+self: super: {
+  a = self.callPackage ./pkgs/by-name/a/a/package.nix { };
+  b = self.callPackage ({ someDrv }: someDrv) { };
+  c = self.callPackage ./pkgs/by-name/c/c/package.nix { };
+  d = self.callPackage ({ someDrv }: someDrv) { };
+}
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/expected b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/expected
new file mode 100644
index 000000000000..349e9ad47c41
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/expected
@@ -0,0 +1,4 @@
+pkgs.a: This attribute is manually defined (most likely in pkgs/top-level/all-packages.nix), which is only allowed if the definition is of the form `pkgs.callPackage pkgs/by-name/a/a/package.nix { ... }` with a non-empty second argument.
+pkgs.b: This is a new top-level package of the form `callPackage ... { }`. Please define it in pkgs/by-name/b/b/package.nix instead. See `pkgs/by-name/README.md` for more details. Since the second `callPackage` argument is `{ }`, no manual `callPackage` (e.g. in `pkgs/top-level/all-packages.nix`) is needed anymore.
+pkgs.c: This attribute is manually defined (most likely in pkgs/top-level/all-packages.nix), which is only allowed if the definition is of the form `pkgs.callPackage pkgs/by-name/c/c/package.nix { ... }` with a non-empty second argument.
+pkgs.d: This is a new top-level package of the form `callPackage ... { }`. Please define it in pkgs/by-name/d/d/package.nix instead. See `pkgs/by-name/README.md` for more details. Since the second `callPackage` argument is `{ }`, no manual `callPackage` (e.g. in `pkgs/top-level/all-packages.nix`) is needed anymore.
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/pkgs/by-name/a/a/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/pkgs/by-name/a/a/package.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/pkgs/by-name/a/a/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/pkgs/by-name/c/c/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/pkgs/by-name/c/c/package.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/pkgs/test/nixpkgs-check-by-name/tests/sorted-order/pkgs/by-name/c/c/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/success/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/success/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/success/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/success/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/symlink-escape/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/uppercase/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/uppercase/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/uppercase/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/uppercase/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }
diff --git a/pkgs/test/nixpkgs-check-by-name/tests/with-readme/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/with-readme/default.nix
index af25d1450122..861260cdca4b 100644
--- a/pkgs/test/nixpkgs-check-by-name/tests/with-readme/default.nix
+++ b/pkgs/test/nixpkgs-check-by-name/tests/with-readme/default.nix
@@ -1 +1 @@
-import ../mock-nixpkgs.nix { root = ./.; }
+import <test-nixpkgs> { root = ./.; }