Merge branch 'nixos-unstable-small' of https://github.com/NixOS/nixpkgs

Conflicts:
	nixpkgs/nixos/modules/services/mail/rss2email.nix
	nixpkgs/pkgs/build-support/go/module.nix

52 files changed, 1312 insertions, 339 deletions

diff --git a/nixpkgs/pkgs/test/buildFHSEnv/default.nix b/nixpkgs/pkgs/test/buildFHSEnv/default.nix
new file mode 100644
index 000000000000..0a355e1aeeac
--- /dev/null
+++ b/nixpkgs/pkgs/test/buildFHSEnv/default.nix
@@ -0,0 +1,84 @@
+{ lib
+, buildFHSEnv
+, runCommand
+, stdenv
+, fetchurl
+, dpkg
+, glibc
+, callPackage
+}:
+
+let
+  getSharedObjectFromDebian = sharedObjectName: src: stdenv.mkDerivation  {
+    name = "${sharedObjectName}-fetcher";
+    inherit src;
+    nativeBuildInputs = [
+      dpkg
+    ];
+    dontBuild = true;
+    dontConfigure = true;
+    dontFixup = true;
+    installPhase = ''
+      echo shared objects found are:
+      ls -l usr/lib/*/
+      cp usr/lib/*/${sharedObjectName} $out
+    '';
+  };
+
+  makeSharedObjectTest = sharedObject: targetPkgs: let
+    lddFHSEnv = buildFHSEnv {
+      name = "ldd-with-ncurses-FHS-env";
+      inherit targetPkgs;
+      runScript = "ldd";
+    };
+    ldd-in-FHS = "${lddFHSEnv}/bin/${lddFHSEnv.name}";
+    ldd = "${lib.getBin glibc}/bin/ldd";
+    find_libFHSEnv = buildFHSEnv {
+      name = "ls-with-ncurses-FHS-env";
+      targetPkgs = p: [
+        p.ncurses5
+      ];
+      runScript = "find /lib/ -executable";
+    };
+    find_lib-in-FHS = "${find_libFHSEnv}/bin/${find_libFHSEnv.name}";
+  in runCommand "FHS-lib-test" {} ''
+    echo original ldd output is:
+    ${ldd} ${sharedObject}
+    lddOutput="$(${ldd-in-FHS} ${sharedObject})"
+    echo ldd output inside FHS is:
+    echo "$lddOutput"
+    if echo $lddOutput | grep -q "not found"; then
+      echo "shared object could not find all dependencies in the FHS!"
+      echo The libraries below where found in the FHS:
+      ${find_lib-in-FHS}
+      exit 1
+    else
+      echo $lddOutput > $out
+    fi
+  '';
+
+in {
+  # This test proves an issue with buildFHSEnv - don't expect it to succeed,
+  # this is discussed in https://github.com/NixOS/nixpkgs/pull/279844 .
+  libtinfo = makeSharedObjectTest (getSharedObjectFromDebian "libedit.so.2.0.70" (fetchurl {
+    url = "mirror://debian/pool/main/libe/libedit/libedit2_3.1-20221030-2_amd64.deb";
+    hash = "sha256-HPFKvycW0yedsS0GV6VzfPcAdKHnHTvfcyBmJePInOY=";
+  })) (p: let
+    ncurses' = p.ncurses.overrideAttrs (old: {
+      configureFlags = old.configureFlags ++ [ "--with-termlib" ];
+      postFixup = "";
+    });
+  in [
+    (ncurses'.override { unicodeSupport = false; })
+    p.libbsd
+  ]);
+
+  liblzma = makeSharedObjectTest (getSharedObjectFromDebian "libxml2.so.2.9.14" (fetchurl {
+    url = "mirror://debian/pool/main/libx/libxml2/libxml2_2.9.14+dfsg-1.3~deb12u1_amd64.deb";
+    hash = "sha256-NbdstwOPwclAIEpPBfM/+3nQJzU85Gk5fZrc+Pmz4ac=";
+  })) (p: [
+    p.xz
+    p.zlib
+    p.icu72
+  ]);
+}
diff --git a/nixpkgs/pkgs/test/cross/default.nix b/nixpkgs/pkgs/test/cross/default.nix
index b4da2de5c5b8..bd233db4cd50 100644
--- a/nixpkgs/pkgs/test/cross/default.nix
+++ b/nixpkgs/pkgs/test/cross/default.nix
@@ -154,7 +154,7 @@ let
     pkgs.pkgsMusl.pkgsCross.gnu64.hello
 
     # Two web browsers -- exercises almost the entire packageset
-    pkgs.pkgsCross.aarch64-multiplatform.qt5.qutebrowser
+    pkgs.pkgsCross.aarch64-multiplatform.qutebrowser-qt5
     pkgs.pkgsCross.aarch64-multiplatform.firefox
 
     # Uses pkgsCross.riscv64-embedded; see https://github.com/NixOS/nixpkgs/issues/267859
diff --git a/nixpkgs/pkgs/test/default.nix b/nixpkgs/pkgs/test/default.nix
index 363b0a2e1519..b89fcc3ecb6d 100644
--- a/nixpkgs/pkgs/test/default.nix
+++ b/nixpkgs/pkgs/test/default.nix
@@ -9,10 +9,14 @@ with pkgs;
       pkgSets = lib.pipe pkgNames [
         (filter (lib.hasPrefix "llvmPackages"))
         (filter (n: n != "rocmPackages.llvm"))
-        # Is a throw alias.
+        # Are throw aliases.
         (filter (n: n != "llvmPackages_rocm"))
         (filter (n: n != "llvmPackages_latest"))
         (filter (n: n != "llvmPackages_git"))
+        (filter (n: n != "llvmPackages_6"))
+        (filter (n: n != "llvmPackages_7"))
+        (filter (n: n != "llvmPackages_8"))
+        (filter (n: n != "llvmPackages_10"))
       ];
       tests = lib.genAttrs pkgSets (name: recurseIntoAttrs {
         clang = callPackage ./cc-wrapper { stdenv = pkgs.${name}.stdenv; };
@@ -64,9 +68,7 @@ with pkgs;
             # libcxxStdenv broken
             # fix in https://github.com/NixOS/nixpkgs/pull/216273
           ] ++ lib.optionals (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64) [
-            (filterAttrs (n: _: n != "llvmPackages_8"))
             (filterAttrs (n: _: n != "llvmPackages_9"))
-            (filterAttrs (n: _: n != "llvmPackages_10"))
           ]);
         in
         toJSON sets;
@@ -167,6 +169,8 @@ with pkgs;
 
   pkgs-lib = recurseIntoAttrs (import ../pkgs-lib/tests { inherit pkgs; });
 
+  buildFHSEnv = recurseIntoAttrs (callPackages ./buildFHSEnv { });
+
   nixpkgs-check-by-name = callPackage ./nixpkgs-check-by-name { };
 
   auto-patchelf-hook = callPackage ./auto-patchelf-hook { };
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/Cargo.lock b/nixpkgs/pkgs/test/nixpkgs-check-by-name/Cargo.lock
index fc3aeb9fd79b..904a9cff0e78 100644
--- a/nixpkgs/pkgs/test/nixpkgs-check-by-name/Cargo.lock
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/Cargo.lock
@@ -214,6 +214,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
 
 [[package]]
+name = "indoc"
+version = "2.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8"
+
+[[package]]
 name = "is-terminal"
 version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -289,10 +295,12 @@ dependencies = [
  "anyhow",
  "clap",
  "colored",
+ "indoc",
  "itertools",
  "lazy_static",
  "regex",
  "rnix",
+ "rowan",
  "serde",
  "serde_json",
  "temp-env",
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/Cargo.toml b/nixpkgs/pkgs/test/nixpkgs-check-by-name/Cargo.toml
index 1e6eaa1106d5..5240cd69f996 100644
--- a/nixpkgs/pkgs/test/nixpkgs-check-by-name/Cargo.toml
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/Cargo.toml
@@ -14,6 +14,8 @@ anyhow = "1.0"
 lazy_static = "1.4.0"
 colored = "2.0.4"
 itertools = "0.11.0"
+rowan = "0.15.11"
 
 [dev-dependencies]
 temp-env = "0.3.5"
+indoc = "2.0.4"
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/scripts/pinned-tool.json b/nixpkgs/pkgs/test/nixpkgs-check-by-name/scripts/pinned-tool.json
index b703ce74771e..4ecb86ff6dcf 100644
--- a/nixpkgs/pkgs/test/nixpkgs-check-by-name/scripts/pinned-tool.json
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/scripts/pinned-tool.json
@@ -1,4 +1,4 @@
 {
-  "rev": "9b19f5e77dd906cb52dade0b7bd280339d2a1f3d",
-  "ci-path": "/nix/store/qlls5ca8q88qpyygg9ddi60gl1nmvpij-nixpkgs-check-by-name"
+  "rev": "f8e2ebd66d097614d51a56a755450d4ae1632df1",
+  "ci-path": "/nix/store/4kv4fyb6x5ivn0qncg7d9i5zhqhzy7bi-nixpkgs-check-by-name"
 }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/eval.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/eval.nix
index 87c54b6444ee..ab1c41e0b145 100644
--- a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/eval.nix
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/eval.nix
@@ -1,5 +1,5 @@
-# Takes a path to nixpkgs and a path to the json-encoded list of attributes to check.
-# Returns an value containing information on each requested attribute,
+# Takes a path to nixpkgs and a path to the json-encoded list of `pkgs/by-name` attributes.
+# Returns a value containing information on all Nixpkgs attributes
 # which is decoded on the Rust side.
 # See ./eval.rs for the meaning of the returned values
 {
@@ -9,33 +9,28 @@
 let
   attrs = builtins.fromJSON (builtins.readFile attrsPath);
 
-  nixpkgsPathLength = builtins.stringLength (toString nixpkgsPath) + 1;
-  removeNixpkgsPrefix = builtins.substring nixpkgsPathLength (-1);
-
-  # We need access to the `callPackage` arguments of each attribute.
-  # The only way to do so is to override `callPackage` with our own version that adds this information to the result,
-  # and then try to access this information.
+  # We need to check whether attributes are defined manually e.g. in
+  # `all-packages.nix`, automatically by the `pkgs/by-name` overlay, or
+  # neither. The only way to do so is to override `callPackage` and
+  # `_internalCallByNamePackageFile` with our own version that adds this
+  # information to the result, and then try to access it.
   overlay = final: prev: {
 
-    # Information for attributes defined using `callPackage`
+    # Adds information to each attribute about whether it's manually defined using `callPackage`
     callPackage = fn: args:
       addVariantInfo (prev.callPackage fn args) {
-        Manual = {
-          path =
-            if builtins.isPath fn then
-              removeNixpkgsPrefix (toString fn)
-            else
-              null;
-          empty_arg =
-            args == { };
-        };
+        # This is a manual definition of the attribute, and it's a callPackage, specifically a semantic callPackage
+        ManualDefinition.is_semantic_call_package = true;
       };
 
-    # Information for attributes that are auto-called from pkgs/by-name.
-    # This internal attribute is only used by pkgs/by-name
+    # Adds information to each attribute about whether it's automatically
+    # defined by the `pkgs/by-name` overlay. This internal attribute is only
+    # used by that overlay.
+    # This overrides the above `callPackage` information (we don't need that
+    # one, since `pkgs/by-name` always uses `callPackage` underneath.
     _internalCallByNamePackageFile = file:
       addVariantInfo (prev._internalCallByNamePackageFile file) {
-        Auto = null;
+        AutoDefinition = null;
       };
 
   };
@@ -50,7 +45,7 @@ let
     else
       # It's very rare that callPackage doesn't return an attribute set, but it can occur.
       # In such a case we can't really return anything sensible that would include the info,
-      # so just don't return the info and let the consumer handle it.
+      # so just don't return the value directly and treat it as if it wasn't a callPackage.
       value;
 
   pkgs = import nixpkgsPath {
@@ -62,29 +57,33 @@ let
     system = "x86_64-linux";
   };
 
-  attrInfo = name: value:
-    if ! builtins.isAttrs value then
-      {
-        NonAttributeSet = null;
-      }
-    else if ! value ? _callPackageVariant then
-      {
-        NonCallPackage = null;
-      }
-    else
-      {
-        CallPackage = {
-          call_package_variant = value._callPackageVariant;
-          is_derivation = pkgs.lib.isDerivation value;
+  # See AttributeInfo in ./eval.rs for the meaning of this
+  attrInfo = name: value: {
+    location = builtins.unsafeGetAttrPos name pkgs;
+    attribute_variant =
+      if ! builtins.isAttrs value then
+        { NonAttributeSet = null; }
+      else
+        {
+          AttributeSet = {
+            is_derivation = pkgs.lib.isDerivation value;
+            definition_variant =
+              if ! value ? _callPackageVariant then
+                { ManualDefinition.is_semantic_call_package = false; }
+              else
+                value._callPackageVariant;
+          };
         };
-      };
+  };
 
+  # Information on all attributes that are in pkgs/by-name.
   byNameAttrs = builtins.listToAttrs (map (name: {
     inherit name;
     value.ByName =
       if ! pkgs ? ${name} then
         { Missing = null; }
       else
+        # Evaluation failures are not allowed, so don't try to catch them
         { Existing = attrInfo name pkgs.${name}; };
   }) attrs);
 
@@ -92,6 +91,8 @@ let
   # We need this to enforce pkgs/by-name for new packages
   nonByNameAttrs = builtins.mapAttrs (name: value:
     let
+      # Packages outside `pkgs/by-name` often fail evaluation,
+      # so we need to handle that
       output = attrInfo name value;
       result = builtins.tryEval (builtins.deepSeq output null);
     in
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/eval.rs b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/eval.rs
index e4584f09d8cd..e90a95533144 100644
--- a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/eval.rs
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/eval.rs
@@ -1,7 +1,10 @@
 use crate::nixpkgs_problem::NixpkgsProblem;
 use crate::ratchet;
 use crate::structure;
+use crate::utils;
+use crate::validation::ResultIteratorExt as _;
 use crate::validation::{self, Validation::Success};
+use crate::NixFileStore;
 use std::path::Path;
 
 use anyhow::Context;
@@ -34,34 +37,43 @@ enum ByNameAttribute {
 }
 
 #[derive(Deserialize)]
-enum AttributeInfo {
-    /// The attribute exists, but its value isn't an attribute set
-    NonAttributeSet,
-    /// The attribute exists, but its value isn't defined using callPackage
-    NonCallPackage,
-    /// The attribute exists and its value is an attribute set
-    CallPackage(CallPackageInfo),
+struct AttributeInfo {
+    /// The location of the attribute as returned by `builtins.unsafeGetAttrPos`
+    location: Option<Location>,
+    attribute_variant: AttributeVariant,
+}
+
+/// The structure returned by a successful `builtins.unsafeGetAttrPos`
+#[derive(Deserialize, Clone, Debug)]
+struct Location {
+    pub file: PathBuf,
+    pub line: usize,
+    pub column: usize,
 }
 
 #[derive(Deserialize)]
-struct CallPackageInfo {
-    call_package_variant: CallPackageVariant,
-    /// Whether the attribute is a derivation (`lib.isDerivation`)
-    is_derivation: bool,
+pub enum AttributeVariant {
+    /// The attribute is not an attribute set, we're limited in the amount of information we can get
+    /// from it (though it's obviously not a derivation)
+    NonAttributeSet,
+    AttributeSet {
+        /// Whether the attribute is a derivation (`lib.isDerivation`)
+        is_derivation: bool,
+        /// The type of callPackage
+        definition_variant: DefinitionVariant,
+    },
 }
 
 #[derive(Deserialize)]
-enum CallPackageVariant {
-    /// The attribute is auto-called as pkgs.callPackage using pkgs/by-name,
-    /// and it is not overridden by a definition in all-packages.nix
-    Auto,
-    /// The attribute is defined as a pkgs.callPackage <path> <args>,
-    /// and overridden by all-packages.nix
-    Manual {
-        /// The <path> argument or None if it's not a path
-        path: Option<PathBuf>,
-        /// true if <args> is { }
-        empty_arg: bool,
+pub enum DefinitionVariant {
+    /// An automatic definition by the `pkgs/by-name` overlay
+    /// Though it's detected using the internal _internalCallByNamePackageFile attribute,
+    /// which can in theory also be used by other code
+    AutoDefinition,
+    /// A manual definition of the attribute, typically in `all-packages.nix`
+    ManualDefinition {
+        /// Whether the attribute is defined as `pkgs.callPackage ...` or something else.
+        is_semantic_call_package: bool,
     },
 }
 
@@ -70,6 +82,7 @@ enum CallPackageVariant {
 /// See the `eval.nix` file for how this is achieved on the Nix side
 pub fn check_values(
     nixpkgs_path: &Path,
+    nix_file_store: &mut NixFileStore,
     package_names: Vec<String>,
     keep_nix_path: bool,
 ) -> validation::Result<ratchet::Nixpkgs> {
@@ -142,149 +155,320 @@ pub fn check_values(
             )
         })?;
 
-    let check_result = validation::sequence(attributes.into_iter().map(
-        |(attribute_name, attribute_value)| {
-            let relative_package_file = structure::relative_file_for_package(&attribute_name);
+    let check_result = validation::sequence(
+        attributes
+            .into_iter()
+            .map(|(attribute_name, attribute_value)| {
+                let check_result = match attribute_value {
+                    Attribute::NonByName(non_by_name_attribute) => handle_non_by_name_attribute(
+                        nixpkgs_path,
+                        nix_file_store,
+                        non_by_name_attribute,
+                    )?,
+                    Attribute::ByName(by_name_attribute) => by_name(
+                        nix_file_store,
+                        nixpkgs_path,
+                        &attribute_name,
+                        by_name_attribute,
+                    )?,
+                };
+                Ok::<_, anyhow::Error>(check_result.map(|value| (attribute_name.clone(), value)))
+            })
+            .collect_vec()?,
+    );
+
+    Ok(check_result.map(|elems| ratchet::Nixpkgs {
+        package_names: elems.iter().map(|(name, _)| name.to_owned()).collect(),
+        package_map: elems.into_iter().collect(),
+    }))
+}
 
-            use ratchet::RatchetState::*;
-            use Attribute::*;
-            use AttributeInfo::*;
-            use ByNameAttribute::*;
-            use CallPackageVariant::*;
-            use NonByNameAttribute::*;
+/// Handles the evaluation result for an attribute in `pkgs/by-name`,
+/// turning it into a validation result.
+fn by_name(
+    nix_file_store: &mut NixFileStore,
+    nixpkgs_path: &Path,
+    attribute_name: &str,
+    by_name_attribute: ByNameAttribute,
+) -> validation::Result<ratchet::Package> {
+    use ratchet::RatchetState::*;
+    use ByNameAttribute::*;
 
-            let check_result = match attribute_value {
-                // The attribute succeeds evaluation and is NOT defined in pkgs/by-name
-                NonByName(EvalSuccess(attribute_info)) => {
-                    let uses_by_name = match attribute_info {
-                        // In these cases the package doesn't qualify for being in pkgs/by-name,
-                        // so the UsesByName ratchet is already as tight as it can be
-                        NonAttributeSet => Success(Tight),
-                        NonCallPackage => Success(Tight),
-                        // This is the case when the `pkgs/by-name`-internal _internalCallByNamePackageFile
-                        // is used for a package outside `pkgs/by-name`
-                        CallPackage(CallPackageInfo {
-                            call_package_variant: Auto,
-                            ..
-                        }) => {
-                            // With the current detection mechanism, this also triggers for aliases
-                            // to pkgs/by-name packages, and there's no good method of
-                            // distinguishing alias vs non-alias.
-                            // Using `config.allowAliases = false` at least currently doesn't work
-                            // because there's nothing preventing people from defining aliases that
-                            // are present even with that disabled.
-                            // In the future we could kind of abuse this behavior to have better
-                            // enforcement of conditional aliases, but for now we just need to not
-                            // give an error.
-                            Success(Tight)
-                        }
-                        // Only derivations can be in pkgs/by-name,
-                        // so this attribute doesn't qualify
-                        CallPackage(CallPackageInfo {
-                            is_derivation: false,
-                            ..
-                        }) => Success(Tight),
+    let relative_package_file = structure::relative_file_for_package(attribute_name);
+    let absolute_package_file = nixpkgs_path.join(&relative_package_file);
 
-                        // The case of an attribute that qualifies:
-                        // - Uses callPackage
-                        // - Is a derivation
-                        CallPackage(CallPackageInfo {
-                            is_derivation: true,
-                            call_package_variant: Manual { path, empty_arg },
-                        }) => Success(Loose(ratchet::UsesByName {
-                            call_package_path: path,
-                            empty_arg,
-                        })),
-                    };
-                    uses_by_name.map(|x| ratchet::Package {
-                        empty_non_auto_called: Tight,
-                        uses_by_name: x,
-                    })
-                }
-                NonByName(EvalFailure) => {
-                    // This is a bit of an odd case: We don't even _know_ whether this attribute
-                    // would qualify for using pkgs/by-name. We can either:
-                    // - Assume it's not using pkgs/by-name, which has the problem that if a
-                    //   package evaluation gets broken temporarily, the fix can remove it from
-                    //   pkgs/by-name again
-                    // - Assume it's using pkgs/by-name already, which has the problem that if a
-                    //   package evaluation gets broken temporarily, fixing it requires a move to
-                    //   pkgs/by-name
-                    // We choose the latter, since we want to move towards pkgs/by-name, not away
-                    // from it
-                    Success(ratchet::Package {
-                        empty_non_auto_called: Tight,
-                        uses_by_name: Tight,
-                    })
-                }
-                ByName(Missing) => NixpkgsProblem::UndefinedAttr {
-                    relative_package_file: relative_package_file.clone(),
-                    package_name: attribute_name.clone(),
-                }
-                .into(),
-                ByName(Existing(NonAttributeSet)) => NixpkgsProblem::NonDerivation {
-                    relative_package_file: relative_package_file.clone(),
-                    package_name: attribute_name.clone(),
-                }
-                .into(),
-                ByName(Existing(NonCallPackage)) => NixpkgsProblem::WrongCallPackage {
-                    relative_package_file: relative_package_file.clone(),
-                    package_name: attribute_name.clone(),
-                }
-                .into(),
-                ByName(Existing(CallPackage(CallPackageInfo {
+    // At this point we know that `pkgs/by-name/fo/foo/package.nix` has to exists.
+    // This match decides whether the attribute `foo` is defined accordingly
+    // and whether a legacy manual definition could be removed
+    let manual_definition_result = match by_name_attribute {
+        // The attribute is missing
+        Missing => {
+            // This indicates a bug in the `pkgs/by-name` overlay, because it's supposed to
+            // automatically defined attributes in `pkgs/by-name`
+            NixpkgsProblem::UndefinedAttr {
+                relative_package_file: relative_package_file.to_owned(),
+                package_name: attribute_name.to_owned(),
+            }
+            .into()
+        }
+        // The attribute exists
+        Existing(AttributeInfo {
+            // But it's not an attribute set, which limits the amount of information we can get
+            // about this attribute (see ./eval.nix)
+            attribute_variant: AttributeVariant::NonAttributeSet,
+            location: _location,
+        }) => {
+            // The only thing we know is that it's definitely not a derivation, since those are
+            // always attribute sets.
+            //
+            // We can't know whether the attribute is automatically or manually defined for sure,
+            // and while we could check the location, the error seems clear enough as is.
+            NixpkgsProblem::NonDerivation {
+                relative_package_file: relative_package_file.to_owned(),
+                package_name: attribute_name.to_owned(),
+            }
+            .into()
+        }
+        // The attribute exists
+        Existing(AttributeInfo {
+            // And it's an attribute set, which allows us to get more information about it
+            attribute_variant:
+                AttributeVariant::AttributeSet {
                     is_derivation,
-                    call_package_variant,
-                }))) => {
-                    let check_result = if !is_derivation {
-                        NixpkgsProblem::NonDerivation {
-                            relative_package_file: relative_package_file.clone(),
-                            package_name: attribute_name.clone(),
+                    definition_variant,
+                },
+            location,
+        }) => {
+            // Only derivations are allowed in `pkgs/by-name`
+            let is_derivation_result = if is_derivation {
+                Success(())
+            } else {
+                NixpkgsProblem::NonDerivation {
+                    relative_package_file: relative_package_file.to_owned(),
+                    package_name: attribute_name.to_owned(),
+                }
+                .into()
+            };
+
+            // If the definition looks correct
+            let variant_result = match definition_variant {
+                // An automatic `callPackage` by the `pkgs/by-name` overlay.
+                // Though this gets detected by checking whether the internal
+                // `_internalCallByNamePackageFile` was used
+                DefinitionVariant::AutoDefinition => {
+                    if let Some(_location) = location {
+                        // Such an automatic definition should definitely not have a location
+                        // Having one indicates that somebody is using `_internalCallByNamePackageFile`,
+                        NixpkgsProblem::InternalCallPackageUsed {
+                            attr_name: attribute_name.to_owned(),
                         }
                         .into()
                     } else {
-                        Success(())
-                    };
-
-                    check_result.and(match &call_package_variant {
-                        Auto => Success(ratchet::Package {
-                            empty_non_auto_called: Tight,
-                            uses_by_name: Tight,
-                        }),
-                        Manual { path, empty_arg } => {
-                            let correct_file = if let Some(call_package_path) = path {
-                                relative_package_file == *call_package_path
-                            } else {
-                                false
-                            };
+                        Success(Tight)
+                    }
+                }
+                // The attribute is manually defined, e.g. in `all-packages.nix`.
+                // This means we need to enforce it to look like this:
+                //   callPackage ../pkgs/by-name/fo/foo/package.nix { ... }
+                DefinitionVariant::ManualDefinition {
+                    is_semantic_call_package,
+                } => {
+                    // We should expect manual definitions to have a location, otherwise we can't
+                    // enforce the expected format
+                    if let Some(location) = location {
+                        // Parse the Nix file in the location and figure out whether it's an
+                        // attribute definition of the form `= callPackage <arg1> <arg2>`,
+                        // returning the arguments if so.
+                        let optional_syntactic_call_package = nix_file_store
+                            .get(&location.file)?
+                            .call_package_argument_info_at(
+                            location.line,
+                            location.column,
+                            // We're passing `pkgs/by-name/fo/foo/package.nix` here, which causes
+                            // the function to verify that `<arg1>` is the same path,
+                            // making `syntactic_call_package.relative_path` end up as `""`
+                            // TODO: This is confusing and should be improved
+                            &absolute_package_file,
+                        )?;
 
-                            if correct_file {
-                                Success(ratchet::Package {
-                                    // Empty arguments for non-auto-called packages are not allowed anymore.
-                                    empty_non_auto_called: if *empty_arg {
-                                        Loose(ratchet::EmptyNonAutoCalled)
+                        // At this point, we completed two different checks for whether it's a
+                        // `callPackage`
+                        match (is_semantic_call_package, optional_syntactic_call_package) {
+                            // Something like `<attr> = { ... }`
+                            // or a `pkgs.callPackage` but with the wrong file
+                            (false, None)
+                            // Something like `<attr> = pythonPackages.callPackage ./pkgs/by-name/...`
+                            | (false, Some(_))
+                            // Something like `<attr> = bar` where `bar = pkgs.callPackage ...`
+                            // or a `callPackage` but with the wrong file
+                            | (true, None) => {
+                                // All of these are not of the expected form, so error out
+                                // TODO: Make error more specific, don't lump everything together
+                                NixpkgsProblem::WrongCallPackage {
+                                      relative_package_file: relative_package_file.to_owned(),
+                                      package_name: attribute_name.to_owned(),
+                                }.into()
+                            }
+                            // Something like `<attr> = pkgs.callPackage ./pkgs/by-name/...`,
+                            // with the correct file
+                            (true, Some(syntactic_call_package)) => {
+                                Success(
+                                    // Manual definitions with empty arguments are not allowed
+                                    // anymore
+                                    if syntactic_call_package.empty_arg {
+                                        Loose(())
                                     } else {
                                         Tight
-                                    },
-                                    uses_by_name: Tight,
-                                })
-                            } else {
-                                NixpkgsProblem::WrongCallPackage {
-                                    relative_package_file: relative_package_file.clone(),
-                                    package_name: attribute_name.clone(),
-                                }
-                                .into()
+                                    }
+                                )
                             }
                         }
-                    })
+                    } else {
+                        // If manual definitions don't have a location, it's likely `mapAttrs`'d
+                        // over, e.g. if it's defined in aliases.nix.
+                        // We can't verify whether its of the expected `callPackage`, so error out
+                        NixpkgsProblem::CannotDetermineAttributeLocation {
+                            attr_name: attribute_name.to_owned(),
+                        }
+                        .into()
+                    }
                 }
             };
-            check_result.map(|value| (attribute_name.clone(), value))
-        },
-    ));
 
-    Ok(check_result.map(|elems| ratchet::Nixpkgs {
-        package_names: elems.iter().map(|(name, _)| name.to_owned()).collect(),
-        package_map: elems.into_iter().collect(),
+            // Independently report problems about whether it's a derivation and the callPackage variant
+            is_derivation_result.and(variant_result)
+        }
+    };
+    Ok(
+        // Packages being checked in this function are _always_ already defined in `pkgs/by-name`,
+        // so instead of repeating ourselves all the time to define `uses_by_name`, just set it
+        // once at the end with a map
+        manual_definition_result.map(|manual_definition| ratchet::Package {
+            manual_definition,
+            uses_by_name: Tight,
+        }),
+    )
+}
+
+/// Handles the evaluation result for an attribute _not_ in `pkgs/by-name`,
+/// turning it into a validation result.
+fn handle_non_by_name_attribute(
+    nixpkgs_path: &Path,
+    nix_file_store: &mut NixFileStore,
+    non_by_name_attribute: NonByNameAttribute,
+) -> validation::Result<ratchet::Package> {
+    use ratchet::RatchetState::*;
+    use NonByNameAttribute::*;
+
+    // The ratchet state whether this attribute uses `pkgs/by-name`.
+    // This is never `Tight`, because we only either:
+    // - Know that the attribute _could_ be migrated to `pkgs/by-name`, which is `Loose`
+    // - Or we're unsure, in which case we use NonApplicable
+    let uses_by_name =
+        // This is a big ol' match on various properties of the attribute
+
+        // First, it needs to succeed evaluation. We can't know whether an attribute could be
+        // migrated to `pkgs/by-name` if it doesn't evaluate, since we need to check that it's a
+        // derivation.
+        //
+        // This only has the minor negative effect that if a PR that breaks evaluation
+        // gets merged, fixing those failures won't force anything into `pkgs/by-name`.
+        //
+        // For now this isn't our problem, but in the future we
+        // might have another check to enforce that evaluation must not be broken.
+        //
+        // The alternative of assuming that failing attributes would have been fit for `pkgs/by-name`
+        // has the problem that if a package evaluation gets broken temporarily,
+        // fixing it requires a move to pkgs/by-name, which could happen more
+        // often and isn't really justified.
+        if let EvalSuccess(AttributeInfo {
+            // We're only interested in attributes that are attribute sets (which includes
+            // derivations). Anything else can't be in `pkgs/by-name`.
+            attribute_variant: AttributeVariant::AttributeSet {
+                // Indeed, we only care about derivations, non-derivation attribute sets can't be
+                // in `pkgs/by-name`
+                is_derivation: true,
+                // Of the two definition variants, really only the manual one makes sense here.
+                // Special cases are:
+                // - Manual aliases to auto-called packages are not treated as manual definitions,
+                //   due to limitations in the semantic callPackage detection. So those should be
+                //   ignored.
+                // - Manual definitions using the internal _internalCallByNamePackageFile are
+                //   not treated as manual definitions, since _internalCallByNamePackageFile is
+                //   used to detect automatic ones. We can't distinguish from the above case, so we
+                //   just need to ignore this one too, even if that internal attribute should never
+                //   be called manually.
+                definition_variant: DefinitionVariant::ManualDefinition { is_semantic_call_package }
+            },
+            // We need the location of the manual definition, because otherwise
+            // we can't figure out whether it's a syntactic callPackage
+            location: Some(location),
+        }) = non_by_name_attribute {
+
+        // Parse the Nix file in the location and figure out whether it's an
+        // attribute definition of the form `= callPackage <arg1> <arg2>`,
+        // returning the arguments if so.
+        let optional_syntactic_call_package = nix_file_store
+            .get(&location.file)?
+            .call_package_argument_info_at(
+                location.line,
+                location.column,
+                // Passing the Nixpkgs path here both checks that the <arg1> is within Nixpkgs, and
+                // strips the absolute Nixpkgs path from it, such that
+                // syntactic_call_package.relative_path is relative to Nixpkgs
+                nixpkgs_path
+            )?;
+
+        // At this point, we completed two different checks for whether it's a
+        // `callPackage`
+        match (is_semantic_call_package, optional_syntactic_call_package) {
+            // Something like `<attr> = { }`
+            (false, None)
+            // Something like `<attr> = pythonPackages.callPackage ...`
+            | (false, Some(_))
+            // Something like `<attr> = bar` where `bar = pkgs.callPackage ...`
+            | (true, None) => {
+                // In all of these cases, it's not possible to migrate the package to `pkgs/by-name`
+                NonApplicable
+            }
+            // Something like `<attr> = pkgs.callPackage ...`
+            (true, Some(syntactic_call_package)) => {
+                // It's only possible to migrate such a definitions if..
+                match syntactic_call_package.relative_path {
+                    Some(ref rel_path) if rel_path.starts_with(utils::BASE_SUBPATH) => {
+                        // ..the path is not already within `pkgs/by-name` like
+                        //
+                        //   foo-variant = callPackage ../by-name/fo/foo/package.nix {
+                        //     someFlag = true;
+                        //   }
+                        //
+                        // While such definitions could be moved to `pkgs/by-name` by using
+                        // `.override { someFlag = true; }` instead, this changes the semantics in
+                        // relation with overlays, so migration is generally not possible.
+                        //
+                        // See also "package variants" in RFC 140:
+                        // https://github.com/NixOS/rfcs/blob/master/rfcs/0140-simple-package-paths.md#package-variants
+                        NonApplicable
+                    }
+                    _ => {
+                        // Otherwise, the path is outside `pkgs/by-name`, which means it can be
+                        // migrated
+                        Loose(syntactic_call_package)
+                    }
+                }
+            }
+        }
+    } else {
+        // This catches all the cases not matched by the above `if let`, falling back to not being
+        // able to migrate such attributes
+        NonApplicable
+    };
+    Ok(Success(ratchet::Package {
+        // Packages being checked in this function _always_ need a manual definition, because
+        // they're not using `pkgs/by-name` which would allow avoiding it.
+        // so instead of repeating ourselves all the time to define `manual_definition`,
+        // just set it once at the end here
+        manual_definition: Tight,
+        uses_by_name,
     }))
 }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/main.rs b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/main.rs
index 8179ec8ded74..0d0ddcd7e632 100644
--- a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/main.rs
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/main.rs
@@ -1,4 +1,6 @@
+use crate::nix_file::NixFileStore;
 mod eval;
+mod nix_file;
 mod nixpkgs_problem;
 mod ratchet;
 mod references;
@@ -116,6 +118,8 @@ pub fn check_nixpkgs<W: io::Write>(
     keep_nix_path: bool,
     error_writer: &mut W,
 ) -> validation::Result<ratchet::Nixpkgs> {
+    let mut nix_file_store = NixFileStore::default();
+
     Ok({
         let nixpkgs_path = nixpkgs_path.canonicalize().with_context(|| {
             format!(
@@ -132,9 +136,9 @@ pub fn check_nixpkgs<W: io::Write>(
             )?;
             Success(ratchet::Nixpkgs::default())
         } else {
-            check_structure(&nixpkgs_path)?.result_map(|package_names|
+            check_structure(&nixpkgs_path, &mut nix_file_store)?.result_map(|package_names|
                 // Only if we could successfully parse the structure, we do the evaluation checks
-                eval::check_values(&nixpkgs_path, package_names, keep_nix_path))?
+                eval::check_values(&nixpkgs_path, &mut nix_file_store, package_names, keep_nix_path))?
         }
     })
 }
@@ -169,7 +173,7 @@ mod tests {
 
     // tempfile::tempdir needs to be wrapped in temp_env lock
     // because it accesses TMPDIR environment variable.
-    fn tempdir() -> anyhow::Result<TempDir> {
+    pub fn tempdir() -> anyhow::Result<TempDir> {
         let empty_list: [(&str, Option<&str>); 0] = [];
         Ok(temp_env::with_vars(empty_list, tempfile::tempdir)?)
     }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/nix_file.rs b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/nix_file.rs
new file mode 100644
index 000000000000..836c5e2dcdda
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/nix_file.rs
@@ -0,0 +1,510 @@
+//! This is a utility module for interacting with the syntax of Nix files
+
+use crate::utils::LineIndex;
+use anyhow::Context;
+use rnix::ast;
+use rnix::ast::Expr;
+use rnix::ast::HasEntry;
+use rnix::SyntaxKind;
+use rowan::ast::AstNode;
+use rowan::TextSize;
+use rowan::TokenAtOffset;
+use std::collections::hash_map::Entry;
+use std::collections::HashMap;
+use std::fs::read_to_string;
+use std::path::Path;
+use std::path::PathBuf;
+
+/// A structure to store parse results of Nix files in memory,
+/// making sure that the same file never has to be parsed twice
+#[derive(Default)]
+pub struct NixFileStore {
+    entries: HashMap<PathBuf, NixFile>,
+}
+
+impl NixFileStore {
+    /// Get the store entry for a Nix file if it exists, otherwise parse the file, insert it into
+    /// the store, and return the value
+    ///
+    /// Note that this function only gives an anyhow::Result::Err for I/O errors.
+    /// A parse error is anyhow::Result::Ok(Result::Err(error))
+    pub fn get(&mut self, path: &Path) -> anyhow::Result<&NixFile> {
+        match self.entries.entry(path.to_owned()) {
+            Entry::Occupied(entry) => Ok(entry.into_mut()),
+            Entry::Vacant(entry) => Ok(entry.insert(NixFile::new(path)?)),
+        }
+    }
+}
+
+/// A structure for storing a successfully parsed Nix file
+pub struct NixFile {
+    /// The parent directory of the Nix file, for more convenient error handling
+    pub parent_dir: PathBuf,
+    /// The path to the file itself, for errors
+    pub path: PathBuf,
+    pub syntax_root: rnix::Root,
+    pub line_index: LineIndex,
+}
+
+impl NixFile {
+    /// Creates a new NixFile, failing for I/O or parse errors
+    fn new(path: impl AsRef<Path>) -> anyhow::Result<NixFile> {
+        let Some(parent_dir) = path.as_ref().parent() else {
+            anyhow::bail!("Could not get parent of path {}", path.as_ref().display())
+        };
+
+        let contents = read_to_string(&path)
+            .with_context(|| format!("Could not read file {}", path.as_ref().display()))?;
+        let line_index = LineIndex::new(&contents);
+
+        // NOTE: There's now another Nixpkgs CI check to make sure all changed Nix files parse
+        // correctly, though that uses mainline Nix instead of rnix, so it doesn't give the same
+        // errors. In the future we should unify these two checks, ideally moving the other CI
+        // check into this tool as well and checking for both mainline Nix and rnix.
+        rnix::Root::parse(&contents)
+            // rnix's ::ok returns Result<_, _> , so no error is thrown away like it would be with
+            // std::result's ::ok
+            .ok()
+            .map(|syntax_root| NixFile {
+                parent_dir: parent_dir.to_path_buf(),
+                path: path.as_ref().to_owned(),
+                syntax_root,
+                line_index,
+            })
+            .with_context(|| format!("Could not parse file {} with rnix", path.as_ref().display()))
+    }
+}
+
+/// Information about callPackage arguments
+#[derive(Debug, PartialEq)]
+pub struct CallPackageArgumentInfo {
+    /// The relative path of the first argument, or `None` if it's not a path.
+    pub relative_path: Option<PathBuf>,
+    /// Whether the second argument is an empty attribute set
+    pub empty_arg: bool,
+}
+
+impl NixFile {
+    /// Returns information about callPackage arguments for an attribute at a specific line/column
+    /// index.
+    /// If the location is not of the form `<attr> = callPackage <arg1> <arg2>;`, `None` is
+    /// returned.
+    /// This function only returns `Err` for problems that can't be caused by the Nix contents,
+    /// but rather problems in this programs code itself.
+    ///
+    /// This is meant to be used with the location returned from `builtins.unsafeGetAttrPos`, e.g.:
+    /// - Create file `default.nix` with contents
+    ///   ```nix
+    ///   self: {
+    ///     foo = self.callPackage ./default.nix { };
+    ///   }
+    ///   ```
+    /// - Evaluate
+    ///   ```nix
+    ///   builtins.unsafeGetAttrPos "foo" (import ./default.nix { })
+    ///   ```
+    ///   results in `{ file = ./default.nix; line = 2; column = 3; }`
+    /// - Get the NixFile for `.file` from a `NixFileStore`
+    /// - Call this function with `.line`, `.column` and `relative_to` as the (absolute) current directory
+    ///
+    /// You'll get back
+    /// ```rust
+    /// Some(CallPackageArgumentInfo { path = Some("default.nix"), empty_arg: true })
+    /// ```
+    ///
+    /// Note that this also returns the same for `pythonPackages.callPackage`. It doesn't make an
+    /// attempt at distinguishing this.
+    pub fn call_package_argument_info_at(
+        &self,
+        line: usize,
+        column: usize,
+        relative_to: &Path,
+    ) -> anyhow::Result<Option<CallPackageArgumentInfo>> {
+        let Some(attrpath_value) = self.attrpath_value_at(line, column)? else {
+            return Ok(None);
+        };
+        self.attrpath_value_call_package_argument_info(attrpath_value, relative_to)
+    }
+
+    // Internal function mainly to make it independently testable
+    fn attrpath_value_at(
+        &self,
+        line: usize,
+        column: usize,
+    ) -> anyhow::Result<Option<ast::AttrpathValue>> {
+        let index = self.line_index.fromlinecolumn(line, column);
+
+        let token_at_offset = self
+            .syntax_root
+            .syntax()
+            .token_at_offset(TextSize::from(index as u32));
+
+        // The token_at_offset function takes indices to mean a location _between_ characters,
+        // which in this case is some spacing followed by the attribute name:
+        //
+        //   foo = 10;
+        //  /\
+        //  This is the token offset, we get both the (newline + indentation) on the left side,
+        //  and the attribute name on the right side.
+        let TokenAtOffset::Between(_space, token) = token_at_offset else {
+            anyhow::bail!("Line {line} column {column} in {} is not the start of a token, but rather {token_at_offset:?}", self.path.display())
+        };
+
+        // token looks like "foo"
+        let Some(node) = token.parent() else {
+            anyhow::bail!(
+                "Token on line {line} column {column} in {} does not have a parent node: {token:?}",
+                self.path.display()
+            )
+        };
+
+        // node looks like "foo"
+        let Some(attrpath_node) = node.parent() else {
+            anyhow::bail!(
+                "Node in {} does not have a parent node: {node:?}",
+                self.path.display()
+            )
+        };
+
+        if attrpath_node.kind() != SyntaxKind::NODE_ATTRPATH {
+            // This can happen for e.g. `inherit foo`, so definitely not a syntactic `callPackage`
+            return Ok(None);
+        }
+        // attrpath_node looks like "foo.bar"
+        let Some(attrpath_value_node) = attrpath_node.parent() else {
+            anyhow::bail!(
+                "Attribute path node in {} does not have a parent node: {attrpath_node:?}",
+                self.path.display()
+            )
+        };
+
+        if !ast::AttrpathValue::can_cast(attrpath_value_node.kind()) {
+            anyhow::bail!(
+                "Node in {} is not an attribute path value node: {attrpath_value_node:?}",
+                self.path.display()
+            )
+        }
+        // attrpath_value_node looks like "foo.bar = 10;"
+
+        // unwrap is fine because we confirmed that we can cast with the above check.
+        // We could avoid this `unwrap` for a `clone`, since `cast` consumes the argument,
+        // but we still need it for the error message when the cast fails.
+        Ok(Some(ast::AttrpathValue::cast(attrpath_value_node).unwrap()))
+    }
+
+    // Internal function mainly to make attrpath_value_at independently testable
+    fn attrpath_value_call_package_argument_info(
+        &self,
+        attrpath_value: ast::AttrpathValue,
+        relative_to: &Path,
+    ) -> anyhow::Result<Option<CallPackageArgumentInfo>> {
+        let Some(attrpath) = attrpath_value.attrpath() else {
+            anyhow::bail!("attrpath value node doesn't have an attrpath: {attrpath_value:?}")
+        };
+
+        // At this point we know it's something like `foo...bar = ...`
+
+        if attrpath.attrs().count() > 1 {
+            // If the attribute path has multiple entries, the left-most entry is an attribute and
+            // can't be a `callPackage`.
+            //
+            // FIXME: `builtins.unsafeGetAttrPos` will return the same position for all attribute
+            // paths and we can't really know which one it is. We could have a case like
+            // `foo.bar = callPackage ... { }` and trying to determine if `bar` is a `callPackage`,
+            // where this is not correct.
+            // However, this case typically doesn't occur anyways,
+            // because top-level packages wouldn't be nested under an attribute set.
+            return Ok(None);
+        }
+        let Some(value) = attrpath_value.value() else {
+            anyhow::bail!("attrpath value node doesn't have a value: {attrpath_value:?}")
+        };
+
+        // At this point we know it's something like `foo = ...`
+
+        let Expr::Apply(apply1) = value else {
+            // Not even a function call, instead something like `foo = null`
+            return Ok(None);
+        };
+        let Some(function1) = apply1.lambda() else {
+            anyhow::bail!("apply node doesn't have a lambda: {apply1:?}")
+        };
+        let Some(arg1) = apply1.argument() else {
+            anyhow::bail!("apply node doesn't have an argument: {apply1:?}")
+        };
+
+        // At this point we know it's something like `foo = <fun> <arg>`.
+        // For a callPackage, `<fun>` would be `callPackage ./file` and `<arg>` would be `{ }`
+
+        let empty_arg = if let Expr::AttrSet(attrset) = arg1 {
+            // We can only statically determine whether the argument is empty if it's an attribute
+            // set _expression_, even though other kind of expressions could evaluate to an attribute
+            // set _value_. But this is what we want anyways
+            attrset.entries().next().is_none()
+        } else {
+            false
+        };
+
+        // Because callPackage takes two curried arguments, the first function needs to be a
+        // function call itself
+        let Expr::Apply(apply2) = function1 else {
+            // Not a callPackage, instead something like `foo = import ./foo`
+            return Ok(None);
+        };
+        let Some(function2) = apply2.lambda() else {
+            anyhow::bail!("apply node doesn't have a lambda: {apply2:?}")
+        };
+        let Some(arg2) = apply2.argument() else {
+            anyhow::bail!("apply node doesn't have an argument: {apply2:?}")
+        };
+
+        // At this point we know it's something like `foo = <fun2> <arg2> <arg1>`.
+        // For a callPackage, `<fun2>` would be `callPackage`, `<arg2>` would be `./file`
+
+        // Check that <arg2> is a path expression
+        let path = if let Expr::Path(actual_path) = arg2 {
+            // Try to statically resolve the path and turn it into a nixpkgs-relative path
+            if let ResolvedPath::Within(p) = self.static_resolve_path(actual_path, relative_to) {
+                Some(p)
+            } else {
+                // We can't statically know an existing path inside Nixpkgs used as <arg2>
+                None
+            }
+        } else {
+            // <arg2> is not a path, but rather e.g. an inline expression
+            None
+        };
+
+        // Check that <fun2> is an identifier, or an attribute path with an identifier at the end
+        let ident = match function2 {
+            Expr::Ident(ident) => {
+                // This means it's something like `foo = callPackage <arg2> <arg1>`
+                ident
+            }
+            Expr::Select(select) => {
+                // This means it's something like `foo = self.callPackage <arg2> <arg1>`.
+                // We also end up here for e.g. `pythonPackages.callPackage`, but the
+                // callPackage-mocking method will take care of not triggering for this case.
+
+                if select.default_expr().is_some() {
+                    // Very odd case, but this would be `foo = self.callPackage or true ./test.nix {}
+                    // (yes this is valid Nix code)
+                    return Ok(None);
+                }
+                let Some(attrpath) = select.attrpath() else {
+                    anyhow::bail!("select node doesn't have an attrpath: {select:?}")
+                };
+                let Some(last) = attrpath.attrs().last() else {
+                    // This case shouldn't be possible, it would be `foo = self. ./test.nix {}`,
+                    // which shouldn't parse
+                    anyhow::bail!("select node has an empty attrpath: {select:?}")
+                };
+                if let ast::Attr::Ident(ident) = last {
+                    ident
+                } else {
+                    // Here it's something like `foo = self."callPackage" /test.nix {}`
+                    // which we're not gonna bother with
+                    return Ok(None);
+                }
+            }
+            // Any other expression we're not gonna treat as callPackage
+            _ => return Ok(None),
+        };
+
+        let Some(token) = ident.ident_token() else {
+            anyhow::bail!("ident node doesn't have a token: {ident:?}")
+        };
+
+        if token.text() == "callPackage" {
+            Ok(Some(CallPackageArgumentInfo {
+                relative_path: path,
+                empty_arg,
+            }))
+        } else {
+            Ok(None)
+        }
+    }
+}
+
+/// The result of trying to statically resolve a Nix path expression
+pub enum ResolvedPath {
+    /// Something like `./foo/${bar}/baz`, can't be known statically
+    Interpolated,
+    /// Something like `<nixpkgs>`, can't be known statically
+    SearchPath,
+    /// Path couldn't be resolved due to an IO error,
+    /// e.g. if the path doesn't exist or you don't have the right permissions
+    Unresolvable(std::io::Error),
+    /// The path is outside the given absolute path
+    Outside,
+    /// The path is within the given absolute path.
+    /// The `PathBuf` is the relative path under the given absolute path.
+    Within(PathBuf),
+}
+
+impl NixFile {
+    /// Statically resolves a Nix path expression and checks that it's within an absolute path
+    ///
+    /// E.g. for the path expression `./bar.nix` in `./foo.nix` and an absolute path of the
+    /// current directory, the function returns `ResolvedPath::Within(./bar.nix)`
+    pub fn static_resolve_path(&self, node: ast::Path, relative_to: &Path) -> ResolvedPath {
+        if node.parts().count() != 1 {
+            // If there's more than 1 interpolated part, it's of the form `./foo/${bar}/baz`.
+            return ResolvedPath::Interpolated;
+        }
+
+        let text = node.to_string();
+
+        if text.starts_with('<') {
+            // A search path like `<nixpkgs>`. There doesn't appear to be better way to detect
+            // these in rnix
+            return ResolvedPath::SearchPath;
+        }
+
+        // Join the file's parent directory and the path expression, then resolve it
+        // FIXME: Expressions like `../../../../foo/bar/baz/qux` or absolute paths
+        // may resolve close to the original file, but may have left the relative_to.
+        // That should be checked more strictly
+        match self.parent_dir.join(Path::new(&text)).canonicalize() {
+            Err(resolution_error) => ResolvedPath::Unresolvable(resolution_error),
+            Ok(resolved) => {
+                // Check if it's within relative_to
+                match resolved.strip_prefix(relative_to) {
+                    Err(_prefix_error) => ResolvedPath::Outside,
+                    Ok(suffix) => ResolvedPath::Within(suffix.to_path_buf()),
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::tests;
+    use indoc::indoc;
+
+    #[test]
+    fn detects_attributes() -> anyhow::Result<()> {
+        let temp_dir = tests::tempdir()?;
+        let file = temp_dir.path().join("file.nix");
+        let contents = indoc! {r#"
+            toInherit: {
+              foo = 1;
+              "bar" = 2;
+              ${"baz"} = 3;
+              "${"qux"}" = 4;
+
+              # A
+              quux
+              # B
+              =
+              # C
+              5
+              # D
+              ;
+              # E
+
+              /**/quuux/**/=/**/5/**/;/*E*/
+
+              inherit toInherit;
+            }
+        "#};
+
+        std::fs::write(&file, contents)?;
+
+        let nix_file = NixFile::new(&file)?;
+
+        // These are builtins.unsafeGetAttrPos locations for the attributes
+        let cases = [
+            (2, 3, Some("foo = 1;")),
+            (3, 3, Some(r#""bar" = 2;"#)),
+            (4, 3, Some(r#"${"baz"} = 3;"#)),
+            (5, 3, Some(r#""${"qux"}" = 4;"#)),
+            (8, 3, Some("quux\n  # B\n  =\n  # C\n  5\n  # D\n  ;")),
+            (17, 7, Some("quuux/**/=/**/5/**/;")),
+            (19, 10, None),
+        ];
+
+        for (line, column, expected_result) in cases {
+            let actual_result = nix_file
+                .attrpath_value_at(line, column)?
+                .map(|node| node.to_string());
+            assert_eq!(actual_result.as_deref(), expected_result);
+        }
+
+        Ok(())
+    }
+
+    #[test]
+    fn detects_call_package() -> anyhow::Result<()> {
+        let temp_dir = tests::tempdir()?;
+        let file = temp_dir.path().join("file.nix");
+        let contents = indoc! {r#"
+            self: with self; {
+              a.sub = null;
+              b = null;
+              c = import ./file.nix;
+              d = import ./file.nix { };
+              e = pythonPackages.callPackage ./file.nix { };
+              f = callPackage ./file.nix { };
+              g = callPackage ({ }: { }) { };
+              h = callPackage ./file.nix { x = 0; };
+              i = callPackage ({ }: { }) (let in { });
+            }
+        "#};
+
+        std::fs::write(&file, contents)?;
+
+        let nix_file = NixFile::new(&file)?;
+
+        let cases = [
+            (2, None),
+            (3, None),
+            (4, None),
+            (5, None),
+            (
+                6,
+                Some(CallPackageArgumentInfo {
+                    relative_path: Some(PathBuf::from("file.nix")),
+                    empty_arg: true,
+                }),
+            ),
+            (
+                7,
+                Some(CallPackageArgumentInfo {
+                    relative_path: Some(PathBuf::from("file.nix")),
+                    empty_arg: true,
+                }),
+            ),
+            (
+                8,
+                Some(CallPackageArgumentInfo {
+                    relative_path: None,
+                    empty_arg: true,
+                }),
+            ),
+            (
+                9,
+                Some(CallPackageArgumentInfo {
+                    relative_path: Some(PathBuf::from("file.nix")),
+                    empty_arg: false,
+                }),
+            ),
+            (
+                10,
+                Some(CallPackageArgumentInfo {
+                    relative_path: None,
+                    empty_arg: false,
+                }),
+            ),
+        ];
+
+        for (line, expected_result) in cases {
+            let actual_result = nix_file.call_package_argument_info_at(line, 3, temp_dir.path())?;
+            assert_eq!(actual_result, expected_result);
+        }
+
+        Ok(())
+    }
+}
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/nixpkgs_problem.rs b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/nixpkgs_problem.rs
index 16ea65deebfc..e13869adaa41 100644
--- a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/nixpkgs_problem.rs
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/nixpkgs_problem.rs
@@ -1,6 +1,5 @@
 use crate::structure;
 use crate::utils::PACKAGE_NIX_FILENAME;
-use rnix::parser::ParseError;
 use std::ffi::OsString;
 use std::fmt;
 use std::io;
@@ -58,11 +57,6 @@ pub enum NixpkgsProblem {
         subpath: PathBuf,
         io_error: io::Error,
     },
-    CouldNotParseNix {
-        relative_package_dir: PathBuf,
-        subpath: PathBuf,
-        error: ParseError,
-    },
     PathInterpolation {
         relative_package_dir: PathBuf,
         subpath: PathBuf,
@@ -98,6 +92,12 @@ pub enum NixpkgsProblem {
         call_package_path: Option<PathBuf>,
         empty_arg: bool,
     },
+    InternalCallPackageUsed {
+        attr_name: String,
+    },
+    CannotDetermineAttributeLocation {
+        attr_name: String,
+    },
 }
 
 impl fmt::Display for NixpkgsProblem {
@@ -184,14 +184,6 @@ impl fmt::Display for NixpkgsProblem {
                     relative_package_dir.display(),
                     subpath.display(),
                 ),
-            NixpkgsProblem::CouldNotParseNix { relative_package_dir, subpath, error } =>
-                write!(
-                    f,
-                    "{}: File {} could not be parsed by rnix: {}",
-                    relative_package_dir.display(),
-                    subpath.display(),
-                    error,
-                ),
             NixpkgsProblem::PathInterpolation { relative_package_dir, subpath, line, text } =>
                 write!(
                     f,
@@ -266,6 +258,16 @@ impl fmt::Display for NixpkgsProblem {
                     structure::relative_file_for_package(package_name).display(),
                 )
             },
-        }
+            NixpkgsProblem::InternalCallPackageUsed { attr_name } =>
+                write!(
+                    f,
+                    "pkgs.{attr_name}: This attribute is defined using `_internalCallByNamePackageFile`, which is an internal function not intended for manual use.",
+                ),
+            NixpkgsProblem::CannotDetermineAttributeLocation { attr_name } =>
+                write!(
+                    f,
+                    "pkgs.{attr_name}: Cannot determine the location of this attribute using `builtins.unsafeGetAttrPos`.",
+                ),
+       }
     }
 }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/ratchet.rs b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/ratchet.rs
index f8c129626cc0..200bf92c516a 100644
--- a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/ratchet.rs
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/ratchet.rs
@@ -2,11 +2,11 @@
 //!
 //! Each type has a `compare` method that validates the ratchet checks for that item.
 
+use crate::nix_file::CallPackageArgumentInfo;
 use crate::nixpkgs_problem::NixpkgsProblem;
 use crate::structure;
 use crate::validation::{self, Validation, Validation::Success};
 use std::collections::HashMap;
-use std::path::PathBuf;
 
 /// The ratchet value for the entirety of Nixpkgs.
 #[derive(Default)]
@@ -33,7 +33,7 @@ impl Nixpkgs {
 /// The ratchet value for a top-level package
 pub struct Package {
     /// The ratchet value for the check for non-auto-called empty arguments
-    pub empty_non_auto_called: RatchetState<EmptyNonAutoCalled>,
+    pub manual_definition: RatchetState<ManualDefinition>,
 
     /// The ratchet value for the check for new packages using pkgs/by-name
     pub uses_by_name: RatchetState<UsesByName>,
@@ -43,10 +43,10 @@ impl Package {
     /// Validates the ratchet checks for a top-level package
     pub fn compare(name: &str, optional_from: Option<&Self>, to: &Self) -> Validation<()> {
         validation::sequence_([
-            RatchetState::<EmptyNonAutoCalled>::compare(
+            RatchetState::<ManualDefinition>::compare(
                 name,
-                optional_from.map(|x| &x.empty_non_auto_called),
-                &to.empty_non_auto_called,
+                optional_from.map(|x| &x.manual_definition),
+                &to.manual_definition,
             ),
             RatchetState::<UsesByName>::compare(
                 name,
@@ -58,55 +58,82 @@ impl Package {
 }
 
 /// The ratchet state of a generic ratchet check.
-pub enum RatchetState<Context> {
+pub enum RatchetState<Ratchet: ToNixpkgsProblem> {
     /// The ratchet is loose, it can be tightened more.
     /// In other words, this is the legacy state we're trying to move away from.
     /// Introducing new instances is not allowed but previous instances will continue to be allowed.
     /// The `Context` is context for error messages in case a new instance of this state is
     /// introduced
-    Loose(Context),
+    Loose(Ratchet::ToContext),
     /// The ratchet is tight, it can't be tightened any further.
     /// This is either because we already use the latest state, or because the ratchet isn't
     /// relevant.
     Tight,
+    /// This ratchet can't be applied.
+    /// State transitions from/to NonApplicable are always allowed
+    NonApplicable,
 }
 
 /// A trait that can convert an attribute-specific error context into a NixpkgsProblem
 pub trait ToNixpkgsProblem {
+    /// Context relating to the Nixpkgs that is being transitioned _to_
+    type ToContext;
+
     /// How to convert an attribute-specific error context into a NixpkgsProblem
-    fn to_nixpkgs_problem(name: &str, context: &Self, existed_before: bool) -> NixpkgsProblem;
+    fn to_nixpkgs_problem(
+        name: &str,
+        optional_from: Option<()>,
+        to: &Self::ToContext,
+    ) -> NixpkgsProblem;
 }
 
 impl<Context: ToNixpkgsProblem> RatchetState<Context> {
     /// Compare the previous ratchet state of an attribute to the new state.
     /// The previous state may be `None` in case the attribute is new.
     fn compare(name: &str, optional_from: Option<&Self>, to: &Self) -> Validation<()> {
-        // If we don't have a previous state, enforce a tight ratchet
-        let from = optional_from.unwrap_or(&RatchetState::Tight);
-        match (from, to) {
-            // Always okay to keep it tight or tighten the ratchet
-            (_, RatchetState::Tight) => Success(()),
-
-            // Grandfathering policy for a loose ratchet
-            (RatchetState::Loose { .. }, RatchetState::Loose { .. }) => Success(()),
-
+        match (optional_from, to) {
             // Loosening a ratchet is now allowed
-            (RatchetState::Tight, RatchetState::Loose(context)) => {
-                Context::to_nixpkgs_problem(name, context, optional_from.is_some()).into()
+            (Some(RatchetState::Tight), RatchetState::Loose(loose_context)) => {
+                Context::to_nixpkgs_problem(name, Some(()), loose_context).into()
             }
+
+            // Introducing a loose ratchet is also not allowed
+            (None, RatchetState::Loose(loose_context)) => {
+                Context::to_nixpkgs_problem(name, None, loose_context).into()
+            }
+
+            // Everything else is allowed, including:
+            // - Loose -> Loose (grandfathering policy for a loose ratchet)
+            // - -> Tight (always okay to keep or make the ratchet tight)
+            // - Anything involving NotApplicable, where we can't really make any good calls
+            _ => Success(()),
         }
     }
 }
 
-/// The ratchet value of an attribute
-/// for the non-auto-called empty argument check of a single.
+/// The ratchet to check whether a top-level attribute has/needs
+/// a manual definition, e.g. in all-packages.nix.
+///
+/// This ratchet is only tight for attributes that:
+/// - Are not defined in `pkgs/by-name`, and rely on a manual definition
+/// - Are defined in `pkgs/by-name` without any manual definition,
+///   (no custom argument overrides)
+/// - Are defined with `pkgs/by-name` with a manual definition that can't be removed
+///   because it provides custom argument overrides
 ///
-/// This checks that packages defined in `pkgs/by-name` cannot be overridden
-/// with an empty second argument like `callPackage ... { }`.
-pub struct EmptyNonAutoCalled;
+/// In comparison, this ratchet is loose for attributes that:
+/// - Are defined in `pkgs/by-name` with a manual definition
+///   that doesn't have any custom argument overrides
+pub enum ManualDefinition {}
+
+impl ToNixpkgsProblem for ManualDefinition {
+    type ToContext = ();
 
-impl ToNixpkgsProblem for EmptyNonAutoCalled {
-    fn to_nixpkgs_problem(name: &str, _context: &Self, _existed_before: bool) -> NixpkgsProblem {
+    fn to_nixpkgs_problem(
+        name: &str,
+        _optional_from: Option<()>,
+        _to: &Self::ToContext,
+    ) -> NixpkgsProblem {
         NixpkgsProblem::WrongCallPackage {
             relative_package_file: structure::relative_file_for_package(name),
             package_name: name.to_owned(),
@@ -119,27 +146,27 @@ impl ToNixpkgsProblem for EmptyNonAutoCalled {
 ///
 /// This checks that all new package defined using callPackage must be defined via pkgs/by-name
 /// It also checks that once a package uses pkgs/by-name, it can't switch back to all-packages.nix
-#[derive(Clone)]
-pub struct UsesByName {
-    /// The first callPackage argument, used for better errors
-    pub call_package_path: Option<PathBuf>,
-    /// Whether the second callPackage argument is empty, used for better errors
-    pub empty_arg: bool,
-}
+pub enum UsesByName {}
 
 impl ToNixpkgsProblem for UsesByName {
-    fn to_nixpkgs_problem(name: &str, a: &Self, existed_before: bool) -> NixpkgsProblem {
-        if existed_before {
+    type ToContext = CallPackageArgumentInfo;
+
+    fn to_nixpkgs_problem(
+        name: &str,
+        optional_from: Option<()>,
+        to: &Self::ToContext,
+    ) -> NixpkgsProblem {
+        if let Some(()) = optional_from {
             NixpkgsProblem::MovedOutOfByName {
                 package_name: name.to_owned(),
-                call_package_path: a.call_package_path.clone(),
-                empty_arg: a.empty_arg,
+                call_package_path: to.relative_path.clone(),
+                empty_arg: to.empty_arg,
             }
         } else {
             NixpkgsProblem::NewPackageNotUsingByName {
                 package_name: name.to_owned(),
-                call_package_path: a.call_package_path.clone(),
-                empty_arg: a.empty_arg,
+                call_package_path: to.relative_path.clone(),
+                empty_arg: to.empty_arg,
             }
         }
     }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/references.rs b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/references.rs
index ce7403afb32d..169e996300ba 100644
--- a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/references.rs
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/references.rs
@@ -1,23 +1,32 @@
 use crate::nixpkgs_problem::NixpkgsProblem;
 use crate::utils;
-use crate::utils::LineIndex;
 use crate::validation::{self, ResultIteratorExt, Validation::Success};
+use crate::NixFileStore;
 
 use anyhow::Context;
-use rnix::{Root, SyntaxKind::NODE_PATH};
+use rowan::ast::AstNode;
 use std::ffi::OsStr;
-use std::fs::read_to_string;
 use std::path::Path;
 
 /// Check that every package directory in pkgs/by-name doesn't link to outside that directory.
 /// Both symlinks and Nix path expressions are checked.
 pub fn check_references(
+    nix_file_store: &mut NixFileStore,
     relative_package_dir: &Path,
     absolute_package_dir: &Path,
 ) -> validation::Result<()> {
-    // The empty argument here is the subpath under the package directory to check
-    // An empty one means the package directory itself
-    check_path(relative_package_dir, absolute_package_dir, Path::new("")).with_context(|| {
+    // The first subpath to check is the package directory itself, which we can represent as an
+    // empty path, since the absolute package directory gets prepended to this.
+    // We don't use `./.` to keep the error messages cleaner
+    // (there's no canonicalisation going on underneath)
+    let subpath = Path::new("");
+    check_path(
+        nix_file_store,
+        relative_package_dir,
+        absolute_package_dir,
+        subpath,
+    )
+    .with_context(|| {
         format!(
             "While checking the references in package directory {}",
             relative_package_dir.display()
@@ -26,7 +35,12 @@ pub fn check_references(
 }
 
 /// Checks for a specific path to not have references outside
+///
+/// The subpath is the relative path within the package directory we're currently checking.
+/// A relative path so that the error messages don't get absolute paths (which are messy in CI).
+/// The absolute package directory gets prepended before doing anything with it though.
 fn check_path(
+    nix_file_store: &mut NixFileStore,
     relative_package_dir: &Path,
     absolute_package_dir: &Path,
     subpath: &Path,
@@ -62,21 +76,27 @@ fn check_path(
             utils::read_dir_sorted(&path)?
                 .into_iter()
                 .map(|entry| {
-                    let entry_subpath = subpath.join(entry.file_name());
-                    check_path(relative_package_dir, absolute_package_dir, &entry_subpath)
-                        .with_context(|| {
-                            format!("Error while recursing into {}", subpath.display())
-                        })
+                    check_path(
+                        nix_file_store,
+                        relative_package_dir,
+                        absolute_package_dir,
+                        &subpath.join(entry.file_name()),
+                    )
                 })
-                .collect_vec()?,
+                .collect_vec()
+                .with_context(|| format!("Error while recursing into {}", subpath.display()))?,
         )
     } else if path.is_file() {
         // Only check Nix files
         if let Some(ext) = path.extension() {
             if ext == OsStr::new("nix") {
-                check_nix_file(relative_package_dir, absolute_package_dir, subpath).with_context(
-                    || format!("Error while checking Nix file {}", subpath.display()),
-                )?
+                check_nix_file(
+                    nix_file_store,
+                    relative_package_dir,
+                    absolute_package_dir,
+                    subpath,
+                )
+                .with_context(|| format!("Error while checking Nix file {}", subpath.display()))?
             } else {
                 Success(())
             }
@@ -92,91 +112,63 @@ fn check_path(
 /// Check whether a nix file contains path expression references pointing outside the package
 /// directory
 fn check_nix_file(
+    nix_file_store: &mut NixFileStore,
     relative_package_dir: &Path,
     absolute_package_dir: &Path,
     subpath: &Path,
 ) -> validation::Result<()> {
     let path = absolute_package_dir.join(subpath);
-    let parent_dir = path
-        .parent()
-        .with_context(|| format!("Could not get parent of path {}", subpath.display()))?;
 
-    let contents = read_to_string(&path)
-        .with_context(|| format!("Could not read file {}", subpath.display()))?;
+    let nix_file = nix_file_store.get(&path)?;
 
-    let root = Root::parse(&contents);
-    if let Some(error) = root.errors().first() {
-        // NOTE: There's now another Nixpkgs CI check to make sure all changed Nix files parse
-        // correctly, though that uses mainline Nix instead of rnix, so it doesn't give the same
-        // errors. In the future we should unify these two checks, ideally moving the other CI
-        // check into this tool as well and checking for both mainline Nix and rnix.
-        return Ok(NixpkgsProblem::CouldNotParseNix {
-            relative_package_dir: relative_package_dir.to_path_buf(),
-            subpath: subpath.to_path_buf(),
-            error: error.clone(),
-        }
-        .into());
-    }
+    Ok(validation::sequence_(
+        nix_file.syntax_root.syntax().descendants().map(|node| {
+            let text = node.text().to_string();
+            let line = nix_file.line_index.line(node.text_range().start().into());
 
-    let line_index = LineIndex::new(&contents);
+            // We're only interested in Path expressions
+            let Some(path) = rnix::ast::Path::cast(node) else {
+                return Success(());
+            };
 
-    Ok(validation::sequence_(root.syntax().descendants().map(
-        |node| {
-            let text = node.text().to_string();
-            let line = line_index.line(node.text_range().start().into());
+            use crate::nix_file::ResolvedPath;
 
-            if node.kind() != NODE_PATH {
-                // We're only interested in Path expressions
-                Success(())
-            } else if node.children().count() != 0 {
-                // Filters out ./foo/${bar}/baz
-                // TODO: We can just check ./foo
-                NixpkgsProblem::PathInterpolation {
+            match nix_file.static_resolve_path(path, absolute_package_dir) {
+                ResolvedPath::Interpolated => NixpkgsProblem::PathInterpolation {
                     relative_package_dir: relative_package_dir.to_path_buf(),
                     subpath: subpath.to_path_buf(),
                     line,
                     text,
                 }
-                .into()
-            } else if text.starts_with('<') {
-                // Filters out search paths like <nixpkgs>
-                NixpkgsProblem::SearchPath {
+                .into(),
+                ResolvedPath::SearchPath => NixpkgsProblem::SearchPath {
                     relative_package_dir: relative_package_dir.to_path_buf(),
                     subpath: subpath.to_path_buf(),
                     line,
                     text,
                 }
-                .into()
-            } else {
-                // Resolves the reference of the Nix path
-                // turning `../baz` inside `/foo/bar/default.nix` to `/foo/baz`
-                match parent_dir.join(Path::new(&text)).canonicalize() {
-                    Ok(target) => {
-                        // Then checking if it's still in the package directory
-                        // No need to handle the case of it being inside the directory, since we scan through the
-                        // entire directory recursively anyways
-                        if let Err(_prefix_error) = target.strip_prefix(absolute_package_dir) {
-                            NixpkgsProblem::OutsidePathReference {
-                                relative_package_dir: relative_package_dir.to_path_buf(),
-                                subpath: subpath.to_path_buf(),
-                                line,
-                                text,
-                            }
-                            .into()
-                        } else {
-                            Success(())
-                        }
-                    }
-                    Err(e) => NixpkgsProblem::UnresolvablePathReference {
-                        relative_package_dir: relative_package_dir.to_path_buf(),
-                        subpath: subpath.to_path_buf(),
-                        line,
-                        text,
-                        io_error: e,
-                    }
-                    .into(),
+                .into(),
+                ResolvedPath::Outside => NixpkgsProblem::OutsidePathReference {
+                    relative_package_dir: relative_package_dir.to_path_buf(),
+                    subpath: subpath.to_path_buf(),
+                    line,
+                    text,
+                }
+                .into(),
+                ResolvedPath::Unresolvable(e) => NixpkgsProblem::UnresolvablePathReference {
+                    relative_package_dir: relative_package_dir.to_path_buf(),
+                    subpath: subpath.to_path_buf(),
+                    line,
+                    text,
+                    io_error: e,
+                }
+                .into(),
+                ResolvedPath::Within(..) => {
+                    // No need to handle the case of it being inside the directory, since we scan through the
+                    // entire directory recursively anyways
+                    Success(())
                 }
             }
-        },
-    )))
+        }),
+    ))
 }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/structure.rs b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/structure.rs
index 4051ca037c9a..9b615dd9969a 100644
--- a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/structure.rs
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/structure.rs
@@ -3,6 +3,7 @@ use crate::references;
 use crate::utils;
 use crate::utils::{BASE_SUBPATH, PACKAGE_NIX_FILENAME};
 use crate::validation::{self, ResultIteratorExt, Validation::Success};
+use crate::NixFileStore;
 use itertools::concat;
 use lazy_static::lazy_static;
 use regex::Regex;
@@ -34,7 +35,10 @@ pub fn relative_file_for_package(package_name: &str) -> PathBuf {
 
 /// Check the structure of Nixpkgs, returning the attribute names that are defined in
 /// `pkgs/by-name`
-pub fn check_structure(path: &Path) -> validation::Result<Vec<String>> {
+pub fn check_structure(
+    path: &Path,
+    nix_file_store: &mut NixFileStore,
+) -> validation::Result<Vec<String>> {
     let base_dir = path.join(BASE_SUBPATH);
 
     let shard_results = utils::read_dir_sorted(&base_dir)?
@@ -88,7 +92,13 @@ pub fn check_structure(path: &Path) -> validation::Result<Vec<String>> {
                 let package_results = entries
                     .into_iter()
                     .map(|package_entry| {
-                        check_package(path, &shard_name, shard_name_valid, package_entry)
+                        check_package(
+                            nix_file_store,
+                            path,
+                            &shard_name,
+                            shard_name_valid,
+                            package_entry,
+                        )
                     })
                     .collect_vec()?;
 
@@ -102,6 +112,7 @@ pub fn check_structure(path: &Path) -> validation::Result<Vec<String>> {
 }
 
 fn check_package(
+    nix_file_store: &mut NixFileStore,
     path: &Path,
     shard_name: &str,
     shard_name_valid: bool,
@@ -161,6 +172,7 @@ fn check_package(
         });
 
         let result = result.and(references::check_references(
+            nix_file_store,
             &relative_package_dir,
             &path.join(&relative_package_dir),
         )?);
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/utils.rs b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/utils.rs
index 7e0198dede42..9a5d12748918 100644
--- a/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/utils.rs
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/src/utils.rs
@@ -35,12 +35,13 @@ impl LineIndex {
         // the vec
         for split in s.split_inclusive('\n') {
             index += split.len();
-            newlines.push(index);
+            newlines.push(index - 1);
         }
         LineIndex { newlines }
     }
 
-    /// Returns the line number for a string index
+    /// Returns the line number for a string index.
+    /// If the index points to a newline, returns the line number before the newline
     pub fn line(&self, index: usize) -> usize {
         match self.newlines.binary_search(&index) {
             // +1 because lines are 1-indexed
@@ -48,4 +49,47 @@ impl LineIndex {
             Err(x) => x + 1,
         }
     }
+
+    /// Returns the string index for a line and column.
+    pub fn fromlinecolumn(&self, line: usize, column: usize) -> usize {
+        // If it's the 1th line, the column is the index
+        if line == 1 {
+            // But columns are 1-indexed
+            column - 1
+        } else {
+            // For the nth line, we add the index of the (n-1)st newline to the column,
+            // and remove one more from the index since arrays are 0-indexed.
+            // Then add the 1-indexed column to get not the newline index itself,
+            // but rather the index of the position on the next line
+            self.newlines[line - 2] + column
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn line_index() {
+        let line_index = LineIndex::new("a\nbc\n\ndef\n");
+
+        let pairs = [
+            (0, 1, 1),
+            (1, 1, 2),
+            (2, 2, 1),
+            (3, 2, 2),
+            (4, 2, 3),
+            (5, 3, 1),
+            (6, 4, 1),
+            (7, 4, 2),
+            (8, 4, 3),
+            (9, 4, 4),
+        ];
+
+        for (index, line, column) in pairs {
+            assert_eq!(line_index.line(index), line);
+            assert_eq!(line_index.fromlinecolumn(line, column), index);
+        }
+    }
 }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/callPackage-syntax/all-packages.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/callPackage-syntax/all-packages.nix
new file mode 100644
index 000000000000..306d719c9e9d
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/callPackage-syntax/all-packages.nix
@@ -0,0 +1,7 @@
+self: super: {
+  set = self.callPackages ({ callPackage }: {
+    foo = callPackage ({ someDrv }: someDrv) { };
+  }) { };
+
+  inherit (self.set) foo;
+}
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/callPackage-syntax/default.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/callPackage-syntax/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/callPackage-syntax/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/callPackage-syntax/pkgs/by-name/README.md b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/callPackage-syntax/pkgs/by-name/README.md
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/callPackage-syntax/pkgs/by-name/README.md
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/all-packages.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/all-packages.nix
index 95478a87fb32..3fbe2d5e51dc 100644
--- a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/all-packages.nix
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/all-packages.nix
@@ -1,3 +1,4 @@
 self: super: {
   foo = self._internalCallByNamePackageFile ./foo.nix;
+  bar = self._internalCallByNamePackageFile ./foo.nix;
 }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/expected b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/expected
new file mode 100644
index 000000000000..404795ee5c79
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/expected
@@ -0,0 +1 @@
+pkgs.foo: This attribute is defined using `_internalCallByNamePackageFile`, which is an internal function not intended for manual use.
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/pkgs/by-name/fo/foo/package.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/pkgs/by-name/fo/foo/package.nix
index a1b92efbbadb..a1b92efbbadb 100644
--- a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/pkgs/by-name/fo/foo/package.nix
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/internalCallPackage/pkgs/by-name/fo/foo/package.nix
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/all-packages.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/all-packages.nix
new file mode 100644
index 000000000000..07b2caaab4e7
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/all-packages.nix
@@ -0,0 +1,10 @@
+self: super: {
+  nonAttributeSet = self.callPackage ({ someDrv }: someDrv) { };
+  nonCallPackage = self.callPackage ({ someDrv }: someDrv) { };
+  internalCallByName = self.callPackage ({ someDrv }: someDrv) { };
+  nonDerivation = self.callPackage ({ someDrv }: someDrv) { };
+
+  onlyMove = self.callPackage ./pkgs/by-name/on/onlyMove/package.nix { };
+
+  noEval = self.callPackage ./pkgs/by-name/no/noEval/package.nix { };
+}
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/all-packages.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/all-packages.nix
new file mode 100644
index 000000000000..75efb5952e7a
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/all-packages.nix
@@ -0,0 +1,9 @@
+self: super: {
+  nonAttributeSet = null;
+  nonCallPackage = self.someDrv;
+  internalCallByName = self._internalCallByNamePackageFile ./some-pkg.nix;
+  nonDerivation = self.callPackage ({ }: { }) { };
+
+  onlyMove = self.callPackage ({ someDrv }: someDrv) { };
+  noEval = throw "foo";
+}
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/default.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/pkgs/by-name/README.md b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/pkgs/by-name/README.md
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/pkgs/by-name/README.md
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/some-pkg.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/some-pkg.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/base/some-pkg.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/default.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/expected b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/expected
new file mode 100644
index 000000000000..29d33f7dbdc0
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/expected
@@ -0,0 +1,2 @@
+pkgs.noEval: This attribute is manually defined (most likely in pkgs/top-level/all-packages.nix), which is only allowed if the definition is of the form `pkgs.callPackage pkgs/by-name/no/noEval/package.nix { ... }` with a non-empty second argument.
+pkgs.onlyMove: This attribute is manually defined (most likely in pkgs/top-level/all-packages.nix), which is only allowed if the definition is of the form `pkgs.callPackage pkgs/by-name/on/onlyMove/package.nix { ... }` with a non-empty second argument.
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/pkgs/by-name/no/noEval/package.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/pkgs/by-name/no/noEval/package.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/pkgs/by-name/no/noEval/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/pkgs/by-name/on/onlyMove/package.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/pkgs/by-name/on/onlyMove/package.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/manual-definition/pkgs/by-name/on/onlyMove/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/all-packages.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/all-packages.nix
index e2831c2d542e..38762c6de1cc 100644
--- a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/all-packages.nix
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/all-packages.nix
@@ -1,3 +1,5 @@
 self: super: {
   iDontEval = throw "I don't eval";
+
+  futureEval = self.callPackage ({ someDrv }: someDrv) { };
 }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/all-packages.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/all-packages.nix
new file mode 100644
index 000000000000..ac763b454eb0
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/all-packages.nix
@@ -0,0 +1,3 @@
+self: super: {
+  futureEval = throw "foo";
+}
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/default.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/pkgs/by-name/README.md b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/pkgs/by-name/README.md
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/base/pkgs/by-name/README.md
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/pkgs/by-name/README.md b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/pkgs/by-name/README.md
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/no-eval/pkgs/by-name/README.md
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/non-syntactical-callPackage-by-name/all-packages.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/non-syntactical-callPackage-by-name/all-packages.nix
new file mode 100644
index 000000000000..3e0ea20c2281
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/non-syntactical-callPackage-by-name/all-packages.nix
@@ -0,0 +1,6 @@
+self: super: {
+
+  bar = (x: x) self.callPackage ./pkgs/by-name/fo/foo/package.nix { someFlag = true; };
+  foo = self.bar;
+
+}
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/non-syntactical-callPackage-by-name/default.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/non-syntactical-callPackage-by-name/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/non-syntactical-callPackage-by-name/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/non-syntactical-callPackage-by-name/expected b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/non-syntactical-callPackage-by-name/expected
new file mode 100644
index 000000000000..9df788191ece
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/non-syntactical-callPackage-by-name/expected
@@ -0,0 +1 @@
+pkgs.foo: This attribute is manually defined (most likely in pkgs/top-level/all-packages.nix), which is only allowed if the definition is of the form `pkgs.callPackage pkgs/by-name/fo/foo/package.nix { ... }` with a non-empty second argument.
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/non-syntactical-callPackage-by-name/pkgs/by-name/fo/foo/package.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/non-syntactical-callPackage-by-name/pkgs/by-name/fo/foo/package.nix
new file mode 100644
index 000000000000..5ad6ea5e24d6
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/non-syntactical-callPackage-by-name/pkgs/by-name/fo/foo/package.nix
@@ -0,0 +1 @@
+{ someDrv, someFlag }: someDrv
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/pkgs/by-name/README.md b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/pkgs/by-name/README.md
index b0d2b34e338a..e69de29bb2d1 100644
--- a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/pkgs/by-name/README.md
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/pkgs/by-name/README.md
@@ -1 +0,0 @@
-(this is just here so the directory can get tracked by git)
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/all-packages.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/all-packages.nix
new file mode 100644
index 000000000000..85f8c6138c5c
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/all-packages.nix
@@ -0,0 +1,5 @@
+self: super: {
+  foo-variant-unvarianted = self.callPackage ./package.nix { };
+
+  foo-variant-new = self.callPackage ./pkgs/by-name/fo/foo/package.nix { };
+}
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/base/all-packages.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/base/all-packages.nix
new file mode 100644
index 000000000000..734604360073
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/base/all-packages.nix
@@ -0,0 +1,3 @@
+self: super: {
+  foo-variant-unvarianted = self.callPackage ./pkgs/by-name/fo/foo/package.nix { };
+}
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/base/default.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/base/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/base/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/base/pkgs/by-name/fo/foo/package.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/base/pkgs/by-name/fo/foo/package.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/base/pkgs/by-name/fo/foo/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/default.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/package.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/package.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/pkgs/by-name/fo/foo/package.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/pkgs/by-name/fo/foo/package.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/package-variants/pkgs/by-name/fo/foo/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/unknown-location/all-packages.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/unknown-location/all-packages.nix
new file mode 100644
index 000000000000..3398e974cb6b
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/unknown-location/all-packages.nix
@@ -0,0 +1,3 @@
+self: super: builtins.mapAttrs (name: value: value) {
+  foo = self.someDrv;
+}
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/unknown-location/default.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/unknown-location/default.nix
new file mode 100644
index 000000000000..861260cdca4b
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/unknown-location/default.nix
@@ -0,0 +1 @@
+import <test-nixpkgs> { root = ./.; }
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/unknown-location/expected b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/unknown-location/expected
new file mode 100644
index 000000000000..2a248c23ab50
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/unknown-location/expected
@@ -0,0 +1 @@
+pkgs.foo: Cannot determine the location of this attribute using `builtins.unsafeGetAttrPos`.
diff --git a/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/unknown-location/pkgs/by-name/fo/foo/package.nix b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/unknown-location/pkgs/by-name/fo/foo/package.nix
new file mode 100644
index 000000000000..a1b92efbbadb
--- /dev/null
+++ b/nixpkgs/pkgs/test/nixpkgs-check-by-name/tests/unknown-location/pkgs/by-name/fo/foo/package.nix
@@ -0,0 +1 @@
+{ someDrv }: someDrv
diff --git a/nixpkgs/pkgs/test/texlive/default.nix b/nixpkgs/pkgs/test/texlive/default.nix
index 12fdd5c45f8b..5f7067543932 100644
--- a/nixpkgs/pkgs/test/texlive/default.nix
+++ b/nixpkgs/pkgs/test/texlive/default.nix
@@ -1,4 +1,4 @@
-{ lib, stdenv, buildEnv, runCommand, fetchurl, file, texlive, writeShellScript, writeText, texliveInfraOnly, texliveSmall, texliveMedium, texliveFull }:
+{ lib, stdenv, buildEnv, runCommand, fetchurl, file, texlive, writeShellScript, writeText, texliveInfraOnly, texliveConTeXt, texliveSmall, texliveMedium, texliveFull }:
 
 rec {
 
@@ -70,10 +70,26 @@ rec {
       \end{document}
     '';
   } ''
-    chktex -v -nall -w1 "$input" 2>&1 | tee "$out"
+    # chktex is supposed to return 2 when it (successfully) finds warnings, but no errors,
+    # see http://git.savannah.nongnu.org/cgit/chktex.git/commit/?id=ec0fb9b58f02a62ff0bfec98b997208e9d7a5998
+    (set +e; chktex -v -nall -w1 "$input" 2>&1; [ $? = 2 ] || exit 1; set -e)  | tee "$out"
+    # also check that the output does indeed contain "One warning printed"
     grep "One warning printed" "$out"
   '';
 
+  context = mkTeXTest {
+    name = "texlive-test-context";
+    format = "context";
+    texLive = texliveConTeXt;
+    text = ''
+      \starttext
+      \startsection[title={ConTeXt test document}]
+        This is an {\em incredibly} simple ConTeXt document.
+      \stopsection
+      \stoptext
+    '';
+  };
+
   dvipng = lib.recurseIntoAttrs {
     # https://github.com/NixOS/nixpkgs/issues/75605
     basic = runCommand "texlive-test-dvipng-basic" {
@@ -414,6 +430,12 @@ rec {
         # crossrefware: require bibtexperllibs under TEXMFROOT
         "bbl2bib" "bibdoiadd" "bibmradd" "biburl2doi" "bibzbladd" "checkcites" "ltx2crossrefxml"
 
+        # epstopdf: requires kpsewhich
+        "epstopdf" "repstopdf"
+
+        # requires kpsewhich
+        "memoize-extract.pl" "memoize-extract.py"
+
         # require other texlive binaries in PATH
         "allcm" "allec" "chkweb" "fontinst" "ht*" "installfont-tl" "kanji-config-updmap-sys" "kanji-config-updmap-user"
         "kpse*" "latexfileversion" "mkocp" "mkofm" "mtxrunjit" "pdftex-quiet" "pslatex" "rumakeindex" "texconfig"
@@ -421,7 +443,7 @@ rec {
 
         # misc luatex binaries searching for luatex in PATH
         "citeproc-lua" "context" "contextjit" "ctanbib" "digestif" "epspdf" "l3build" "luafindfont" "luaotfload-tool"
-        "luatools" "make4ht" "pmxchords" "tex4ebook" "texdoc" "texlogsieve" "xindex"
+        "luatools" "make4ht" "pmxchords" "tex4ebook" "texblend" "texdoc" "texfindpkg" "texlogsieve" "xindex"
 
         # requires full TEXMFROOT (e.g. for config)
         "mktexfmt" "mktexmf" "mktexpk" "mktextfm" "psnup" "psresize" "pstops" "tlmgr" "updmap" "webquiz"
@@ -500,6 +522,13 @@ rec {
           args=
           ignoreExitCode=
           binCount=$((binCount + 1))
+
+          # ignore non-executable files (such as context.lua)
+          if [[ ! -x "$bin" ]] ; then
+            ignoredCount=$((ignoredCount + 1))
+            continue
+          fi
+
           case "$base" in
             ${lib.concatStringsSep "|" ignored})
               ignoredCount=$((ignoredCount + 1))
@@ -572,6 +601,7 @@ rec {
         (pkg: ''
           for bin in '${pkg.outPath}'/bin/* ; do
             grep -I -q . "$bin" || continue  # ignore binary files
+            [[ -x "$bin" ]] || continue # ignore non-executable files (such as context.lua)
             scriptCount=$((scriptCount + 1))
             read -r cmdline < "$bin"
             read -r interp <<< "$cmdline"