Merge commit '1a57d96edd156958b12782e8c8b6a374142a7248'

15 files changed, 454 insertions, 80 deletions

diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/default.nix b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/default.nix
index 2cfaa69a4c8b..e6ab1f218795 100644
--- a/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/default.nix
+++ b/nixpkgs/pkgs/development/interpreters/python/cpython/2.7/default.nix
@@ -18,12 +18,17 @@
 , ucsEncoding ? 4
 # For the Python package set
 , packageOverrides ? (self: super: {})
-, buildPackages
+, pkgsBuildBuild
+, pkgsBuildHost
+, pkgsBuildTarget
+, pkgsHostHost
+, pkgsTargetTarget
 , sourceVersion
 , sha256
 , passthruFun
 , static ? false
 , enableOptimizations ? (!stdenv.isDarwin)
+, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
 }:
 
 assert x11Support -> tcl != null
@@ -34,8 +39,8 @@ assert x11Support -> tcl != null
 with stdenv.lib;
 
 let
-
-  pythonForBuild = buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"};
+  buildPackages = pkgsBuildHost;
+  inherit (passthru) pythonForBuild;
 
   passthru = passthruFun rec {
     inherit self sourceVersion packageOverrides;
@@ -44,7 +49,12 @@ let
     executable = libPrefix;
     pythonVersion = with sourceVersion; "${major}.${minor}";
     sitePackages = "lib/${libPrefix}/site-packages";
-    inherit hasDistutilsCxxPatch pythonForBuild;
+    inherit hasDistutilsCxxPatch;
+    pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr};
+    pythonOnBuildForHost = pkgsBuildHost.${pythonAttr};
+    pythonOnBuildForTarget = pkgsBuildTarget.${pythonAttr};
+    pythonOnHostForHost = pkgsHostHost.${pythonAttr};
+    pythonOnTargetForTarget = pkgsTargetTarget.${pythonAttr} or {};
   } // {
     inherit ucsEncoding;
   };
diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/3.10/no-ldconfig.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/3.10/no-ldconfig.patch
new file mode 100644
index 000000000000..c259aed72b99
--- /dev/null
+++ b/nixpkgs/pkgs/development/interpreters/python/cpython/3.10/no-ldconfig.patch
@@ -0,0 +1,107 @@
+From 084c6dd6352077e64f10cf7aa168f95d800f3819 Mon Sep 17 00:00:00 2001
+From: Jonathan Ringer <jonringer117@gmail.com>
+Date: Mon, 9 Nov 2020 10:24:35 -0800
+Subject: [PATCH] CPython: Don't use ldconfig
+
+---
+ Lib/ctypes/util.py | 77 ++--------------------------------------------
+ 1 file changed, 2 insertions(+), 75 deletions(-)
+
+diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py
+index 0c2510e..7fb98af 100644
+--- a/Lib/ctypes/util.py
++++ b/Lib/ctypes/util.py
+@@ -100,53 +100,7 @@ elif os.name == "posix":
+             return thefile.read(4) == elf_header
+ 
+     def _findLib_gcc(name):
+-        # Run GCC's linker with the -t (aka --trace) option and examine the
+-        # library name it prints out. The GCC command will fail because we
+-        # haven't supplied a proper program with main(), but that does not
+-        # matter.
+-        expr = os.fsencode(r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name))
+-
+-        c_compiler = shutil.which('gcc')
+-        if not c_compiler:
+-            c_compiler = shutil.which('cc')
+-        if not c_compiler:
+-            # No C compiler available, give up
+-            return None
+-
+-        temp = tempfile.NamedTemporaryFile()
+-        try:
+-            args = [c_compiler, '-Wl,-t', '-o', temp.name, '-l' + name]
+-
+-            env = dict(os.environ)
+-            env['LC_ALL'] = 'C'
+-            env['LANG'] = 'C'
+-            try:
+-                proc = subprocess.Popen(args,
+-                                        stdout=subprocess.PIPE,
+-                                        stderr=subprocess.STDOUT,
+-                                        env=env)
+-            except OSError:  # E.g. bad executable
+-                return None
+-            with proc:
+-                trace = proc.stdout.read()
+-        finally:
+-            try:
+-                temp.close()
+-            except FileNotFoundError:
+-                # Raised if the file was already removed, which is the normal
+-                # behaviour of GCC if linking fails
+-                pass
+-        res = re.findall(expr, trace)
+-        if not res:
+-            return None
+-
+-        for file in res:
+-            # Check if the given file is an elf file: gcc can report
+-            # some files that are linker scripts and not actual
+-            # shared objects. See bpo-41976 for more details
+-            if not _is_elf(file):
+-                continue
+-            return os.fsdecode(file)
++        return None
+ 
+ 
+     if sys.platform == "sunos5":
+@@ -268,34 +222,7 @@ elif os.name == "posix":
+     else:
+ 
+         def _findSoname_ldconfig(name):
+-            import struct
+-            if struct.calcsize('l') == 4:
+-                machine = os.uname().machine + '-32'
+-            else:
+-                machine = os.uname().machine + '-64'
+-            mach_map = {
+-                'x86_64-64': 'libc6,x86-64',
+-                'ppc64-64': 'libc6,64bit',
+-                'sparc64-64': 'libc6,64bit',
+-                's390x-64': 'libc6,64bit',
+-                'ia64-64': 'libc6,IA-64',
+-                }
+-            abi_type = mach_map.get(machine, 'libc6')
+-
+-            # XXX assuming GLIBC's ldconfig (with option -p)
+-            regex = r'\s+(lib%s\.[^\s]+)\s+\(%s'
+-            regex = os.fsencode(regex % (re.escape(name), abi_type))
+-            try:
+-                with subprocess.Popen(['/sbin/ldconfig', '-p'],
+-                                      stdin=subprocess.DEVNULL,
+-                                      stderr=subprocess.DEVNULL,
+-                                      stdout=subprocess.PIPE,
+-                                      env={'LC_ALL': 'C', 'LANG': 'C'}) as p:
+-                    res = re.search(regex, p.stdout.read())
+-                    if res:
+-                        return os.fsdecode(res.group(1))
+-            except OSError:
+-                pass
++            return None
+ 
+         def _findLib_ld(name):
+             # See issue #9998 for why this is needed
+-- 
+2.28.0
+
diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/3.7/find_library.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/3.7/find_library.patch
new file mode 100644
index 000000000000..97fb66662d03
--- /dev/null
+++ b/nixpkgs/pkgs/development/interpreters/python/cpython/3.7/find_library.patch
@@ -0,0 +1,105 @@
+From 9b5a023a5dc3127da15253f7acad71019395ebe1 Mon Sep 17 00:00:00 2001
+From: Pablo Galindo <Pablogsal@gmail.com>
+Date: Thu, 8 Oct 2020 19:50:37 +0100
+Subject: [PATCH] [3.7] bpo-41976: Fix the fallback to gcc of
+ ctypes.util.find_library when using gcc>9 (GH-22598). (GH-22601)
+
+(cherry picked from commit 27ac19cca2c639caaf6fedf3632fe6beb265f24f)
+
+Co-authored-by: Pablo Galindo <Pablogsal@gmail.com>
+---
+ Lib/ctypes/test/test_find.py                  | 12 ++++++-
+ Lib/ctypes/util.py                            | 32 +++++++++++++++----
+ .../2020-10-08-18-22-28.bpo-41976.Svm0wb.rst  |  3 ++
+ 3 files changed, 39 insertions(+), 8 deletions(-)
+ create mode 100644 Misc/NEWS.d/next/Library/2020-10-08-18-22-28.bpo-41976.Svm0wb.rst
+
+diff --git a/Lib/ctypes/test/test_find.py b/Lib/ctypes/test/test_find.py
+index b99fdcba7b28f..92ac1840ad7d4 100644
+--- a/Lib/ctypes/test/test_find.py
++++ b/Lib/ctypes/test/test_find.py
+@@ -1,4 +1,5 @@
+ import unittest
++import unittest.mock
+ import os.path
+ import sys
+ import test.support
+@@ -72,7 +73,7 @@ def test_shell_injection(self):
+ 
+ @unittest.skipUnless(sys.platform.startswith('linux'),
+                      'Test only valid for Linux')
+-class LibPathFindTest(unittest.TestCase):
++class FindLibraryLinux(unittest.TestCase):
+     def test_find_on_libpath(self):
+         import subprocess
+         import tempfile
+@@ -111,6 +112,15 @@ def test_find_on_libpath(self):
+                 # LD_LIBRARY_PATH)
+                 self.assertEqual(find_library(libname), 'lib%s.so' % libname)
+ 
++    def test_find_library_with_gcc(self):
++        with unittest.mock.patch("ctypes.util._findSoname_ldconfig", lambda *args: None):
++            self.assertNotEqual(find_library('c'), None)
++
++    def test_find_library_with_ld(self):
++        with unittest.mock.patch("ctypes.util._findSoname_ldconfig", lambda *args: None), \
++             unittest.mock.patch("ctypes.util._findLib_gcc", lambda *args: None):
++            self.assertNotEqual(find_library('c'), None)
++
+ 
+ if __name__ == "__main__":
+     unittest.main()
+diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py
+index 97973bce001d9..0c2510e1619c8 100644
+--- a/Lib/ctypes/util.py
++++ b/Lib/ctypes/util.py
+@@ -93,6 +93,12 @@ def find_library(name):
+     # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump
+     import re, tempfile
+ 
++    def _is_elf(filename):
++        "Return True if the given file is an ELF file"
++        elf_header = b'\x7fELF'
++        with open(filename, 'br') as thefile:
++            return thefile.read(4) == elf_header
++
+     def _findLib_gcc(name):
+         # Run GCC's linker with the -t (aka --trace) option and examine the
+         # library name it prints out. The GCC command will fail because we
+@@ -299,17 +312,22 @@ def _findLib_ld(name):
+                                      stderr=subprocess.PIPE,
+                                      universal_newlines=True)
+                 out, _ = p.communicate()
+-                res = re.search(expr, os.fsdecode(out))
+-                if res:
+-                    result = res.group(0)
+-            except Exception as e:
++                res = re.findall(expr, os.fsdecode(out))
++                for file in res:
++                    # Check if the given file is an elf file: gcc can report
++                    # some files that are linker scripts and not actual
++                    # shared objects. See bpo-41976 for more details
++                    if not _is_elf(file):
++                        continue
++                    return os.fsdecode(file)
++            except Exception:
+                 pass  # result will be None
+             return result
+ 
+         def find_library(name):
+             # See issue #9998
+             return _findSoname_ldconfig(name) or \
+-                   _get_soname(_findLib_gcc(name) or _findLib_ld(name))
++                   _get_soname(_findLib_gcc(name)) or _get_soname(_findLib_ld(name))
+ 
+ ################################################################
+ # test code
+diff --git a/Misc/NEWS.d/next/Library/2020-10-08-18-22-28.bpo-41976.Svm0wb.rst b/Misc/NEWS.d/next/Library/2020-10-08-18-22-28.bpo-41976.Svm0wb.rst
+new file mode 100644
+index 0000000000000..c8b3fc771845e
+--- /dev/null
++++ b/Misc/NEWS.d/next/Library/2020-10-08-18-22-28.bpo-41976.Svm0wb.rst
+@@ -0,0 +1,3 @@
++Fixed a bug that was causing :func:`ctypes.util.find_library` to return
++``None`` when triying to locate a library in an environment when gcc>=9 is
++available and ``ldconfig`` is not. Patch by Pablo Galindo
diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/3.9/no-ldconfig.patch b/nixpkgs/pkgs/development/interpreters/python/cpython/3.9/no-ldconfig.patch
index a1f9d68eb166..41d3ab52345b 100644
--- a/nixpkgs/pkgs/development/interpreters/python/cpython/3.9/no-ldconfig.patch
+++ b/nixpkgs/pkgs/development/interpreters/python/cpython/3.9/no-ldconfig.patch
@@ -1,19 +1,19 @@
-From 597e73f2a4b2f0b508127931b36d5540d6941823 Mon Sep 17 00:00:00 2001
-From: Frederik Rietdijk <fridh@fridh.nl>
-Date: Mon, 28 Aug 2017 09:24:06 +0200
+From 66f492d2eda94bd64db833839a325caf6ba0fed5 Mon Sep 17 00:00:00 2001
+From: Greg Roodt <greg@canva.com>
+Date: Wed, 9 Dec 2020 17:59:24 +1100
 Subject: [PATCH] Don't use ldconfig
 
 ---
- Lib/ctypes/util.py | 70 ++----------------------------------------------------
- 1 file changed, 2 insertions(+), 68 deletions(-)
+ Lib/ctypes/util.py | 77 ++--------------------------------------------
+ 1 file changed, 2 insertions(+), 75 deletions(-)
 
 diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py
-index 5e8b31a854..7b45ce6c15 100644
+index 0c2510e161..7fb98af308 100644
 --- a/Lib/ctypes/util.py
 +++ b/Lib/ctypes/util.py
-@@ -94,46 +94,7 @@ elif os.name == "posix":
-     import re, tempfile
- 
+@@ -100,53 +100,7 @@ elif os.name == "posix":
+             return thefile.read(4) == elf_header
+
      def _findLib_gcc(name):
 -        # Run GCC's linker with the -t (aka --trace) option and examine the
 -        # library name it prints out. The GCC command will fail because we
@@ -51,17 +51,24 @@ index 5e8b31a854..7b45ce6c15 100644
 -                # Raised if the file was already removed, which is the normal
 -                # behaviour of GCC if linking fails
 -                pass
--        res = re.search(expr, trace)
+-        res = re.findall(expr, trace)
 -        if not res:
 -            return None
--        return os.fsdecode(res.group(0))
+-
+-        for file in res:
+-            # Check if the given file is an elf file: gcc can report
+-            # some files that are linker scripts and not actual
+-            # shared objects. See bpo-41976 for more details
+-            if not _is_elf(file):
+-                continue
+-            return os.fsdecode(file)
 +        return None
- 
- 
+
+
      if sys.platform == "sunos5":
-@@ -255,34 +216,7 @@ elif os.name == "posix":
+@@ -268,34 +222,7 @@ elif os.name == "posix":
      else:
- 
+
          def _findSoname_ldconfig(name):
 -            import struct
 -            if struct.calcsize('l') == 4:
@@ -92,9 +99,8 @@ index 5e8b31a854..7b45ce6c15 100644
 -            except OSError:
 -                pass
 +            return None
- 
+
          def _findLib_ld(name):
              # See issue #9998 for why this is needed
--- 
-2.15.0
-
+--
+2.24.3 (Apple Git-128)
diff --git a/nixpkgs/pkgs/development/interpreters/python/cpython/default.nix b/nixpkgs/pkgs/development/interpreters/python/cpython/default.nix
index b25d613eb7f3..c67ede82003a 100644
--- a/nixpkgs/pkgs/development/interpreters/python/cpython/default.nix
+++ b/nixpkgs/pkgs/development/interpreters/python/cpython/default.nix
@@ -14,12 +14,16 @@
 , self
 , configd
 , autoreconfHook
+, autoconf-archive
 , python-setup-hook
 , nukeReferences
 # For the Python package set
 , packageOverrides ? (self: super: {})
-, buildPackages
-, pythonForBuild ? buildPackages.${"python${sourceVersion.major}${sourceVersion.minor}"}
+, pkgsBuildBuild
+, pkgsBuildHost
+, pkgsBuildTarget
+, pkgsHostHost
+, pkgsTargetTarget
 , sourceVersion
 , sha256
 , passthruFun
@@ -35,6 +39,7 @@
 # Not using optimizations on Darwin
 # configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found.
 , enableOptimizations ? (!stdenv.isDarwin)
+, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
 }:
 
 # Note: this package is used for bootstrapping fetchurl, and thus
@@ -52,6 +57,8 @@ assert bluezSupport -> bluez != null;
 with stdenv.lib;
 
 let
+  buildPackages = pkgsBuildHost;
+  inherit (passthru) pythonForBuild;
 
   passthru = passthruFun rec {
     inherit self sourceVersion packageOverrides;
@@ -60,13 +67,20 @@ let
     executable = libPrefix;
     pythonVersion = with sourceVersion; "${major}.${minor}";
     sitePackages = "lib/${libPrefix}/site-packages";
-    inherit hasDistutilsCxxPatch pythonForBuild;
+    inherit hasDistutilsCxxPatch;
+    pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr};
+    pythonOnBuildForHost = pkgsBuildHost.${pythonAttr};
+    pythonOnBuildForTarget = pkgsBuildTarget.${pythonAttr};
+    pythonOnHostForHost = pkgsHostHost.${pythonAttr};
+    pythonOnTargetForTarget = pkgsTargetTarget.${pythonAttr} or {};
   };
 
   version = with sourceVersion; "${major}.${minor}.${patch}${suffix}";
 
   nativeBuildInputs = optionals (!stdenv.isDarwin) [
     autoreconfHook
+  ] ++ optionals (!stdenv.isDarwin && passthru.pythonAtLeast "3.10") [
+    autoconf-archive # needed for AX_CHECK_COMPILE_FLAG
   ] ++ [
     nukeReferences
   ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
@@ -82,8 +96,6 @@ let
 
   hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
 
-  inherit pythonForBuild;
-
   pythonForBuildInterpreter = if stdenv.hostPlatform == stdenv.buildPlatform then
     "$out/bin/python"
   else pythonForBuild.interpreter;
@@ -153,6 +165,9 @@ in with passthru; stdenv.mkDerivation {
   ] ++ [
     # LDSHARED now uses $CC instead of gcc. Fixes cross-compilation of extension modules.
     ./3.8/0001-On-all-posix-systems-not-just-Darwin-set-LDSHARED-if.patch
+  ] ++ optionals (isPy37 || isPy38) [
+    # Backport a fix for ctypes.util.find_library.
+    ./3.7/find_library.patch
   ];
 
   postPatch = ''
@@ -291,13 +306,6 @@ in with passthru; stdenv.mkDerivation {
     find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
     '' + optionalString stripBytecode ''
     find $out -type d -name __pycache__ -print0 | xargs -0 -I {} rm -rf "{}"
-    '' + ''
-    # *strip* shebang from libpython gdb script - it should be dual-syntax and
-    # interpretable by whatever python the gdb in question is using, which may
-    # not even match the major version of this python. doing this after the
-    # bytecode compilations for the same reason.
-    mkdir -p $out/share/gdb
-    sed '/^#!/d' Tools/gdb/libpython.py > $out/share/gdb/libpython.py
   '';
 
   preFixup = stdenv.lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
@@ -315,8 +323,6 @@ in with passthru; stdenv.mkDerivation {
     pythonForBuild buildPackages.bash
   ];
 
-  separateDebugInfo = true;
-
   inherit passthru;
 
   enableParallelBuilding = true;
diff --git a/nixpkgs/pkgs/development/interpreters/python/default.nix b/nixpkgs/pkgs/development/interpreters/python/default.nix
index ce357b6f58e3..2982cdc8856d 100644
--- a/nixpkgs/pkgs/development/interpreters/python/default.nix
+++ b/nixpkgs/pkgs/development/interpreters/python/default.nix
@@ -14,13 +14,70 @@ with pkgs;
     , packageOverrides
     , sitePackages
     , hasDistutilsCxxPatch
-    , pythonForBuild
-    , self
+    , pythonOnBuildForBuild
+    , pythonOnBuildForHost
+    , pythonOnBuildForTarget
+    , pythonOnHostForHost
+    , pythonOnTargetForTarget
+    , self # is pythonOnHostForTarget
     }: let
-      pythonPackages = callPackage ../../../top-level/python-packages.nix {
-        python = self;
-        overrides = packageOverrides;
-      };
+      pythonPackages = callPackage
+        ({ pkgs, stdenv, python, overrides }: let
+          pythonPackagesFun = import ../../../top-level/python-packages.nix {
+            inherit stdenv pkgs;
+            python = self;
+          };
+          otherSplices = {
+            selfBuildBuild = pythonOnBuildForBuild.pkgs;
+            selfBuildHost = pythonOnBuildForHost.pkgs;
+            selfBuildTarget = pythonOnBuildForTarget.pkgs;
+            selfHostHost = pythonOnHostForHost.pkgs;
+            selfTargetTarget = pythonOnTargetForTarget.pkgs or {}; # There is no Python TargetTarget.
+          };
+          keep = self: {
+            # TODO maybe only define these here so nothing is needed to be kept in sync.
+            inherit (self)
+              isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy3k isPyPy pythonAtLeast pythonOlder
+              python bootstrapped-pip buildPythonPackage buildPythonApplication
+              fetchPypi
+              hasPythonModule requiredPythonModules makePythonPath disabledIf
+              toPythonModule toPythonApplication
+              buildSetupcfg
+
+              eggUnpackHook
+              eggBuildHook
+              eggInstallHook
+              flitBuildHook
+              pipBuildHook
+              pipInstallHook
+              pytestCheckHook
+              pythonCatchConflictsHook
+              pythonImportsCheckHook
+              pythonNamespacesHook
+              pythonRecompileBytecodeHook
+              pythonRemoveBinBytecodeHook
+              pythonRemoveTestsDirHook
+              setuptoolsBuildHook
+              setuptoolsCheckHook
+              venvShellHook
+              wheelUnpackHook
+
+              wrapPython
+
+              pythonPackages
+
+              recursivePthLoader
+            ;
+          };
+        in lib.makeScopeWithSplicing
+          pkgs.splicePackages
+          pkgs.newScope
+          otherSplices
+          keep
+          (lib.extends overrides pythonPackagesFun))
+        {
+          overrides = packageOverrides;
+        };
     in rec {
         isPy27 = pythonVersion == "2.7";
         isPy35 = pythonVersion == "3.5";
@@ -28,6 +85,7 @@ with pkgs;
         isPy37 = pythonVersion == "3.7";
         isPy38 = pythonVersion == "3.8";
         isPy39 = pythonVersion == "3.9";
+        isPy310 = pythonVersion == "3.10";
         isPy2 = lib.strings.substring 0 1 pythonVersion == "2";
         isPy3 = lib.strings.substring 0 1 pythonVersion == "3";
         isPy3k = isPy3;
@@ -41,13 +99,15 @@ with pkgs;
         inherit sourceVersion packageOverrides;
         pythonAtLeast = lib.versionAtLeast pythonVersion;
         pythonOlder = lib.versionOlder pythonVersion;
-        inherit hasDistutilsCxxPatch pythonForBuild;
+        inherit hasDistutilsCxxPatch;
+        # TODO: rename to pythonOnBuild
+        # Not done immediately because its likely used outside Nixpkgs.
+        pythonForBuild = pythonOnBuildForHost.override { inherit packageOverrides; self = pythonForBuild; };
 
         tests = callPackage ./tests.nix {
           python = self;
         };
   };
-
 in {
 
   python27 = callPackage ./cpython/2.7 {
@@ -94,10 +154,10 @@ in {
     sourceVersion = {
       major = "3";
       minor = "8";
-      patch = "5";
+      patch = "6";
       suffix = "";
     };
-    sha256 = "1c43dbv9lvlp3ynqmgdi4rh8q94swanhqarqrdx62zmigpakw073";
+    sha256 = "qeC3nSeqBW65zOjWOkJ7X5urFGXe4/lC3P2yWoL0q4o=";
     inherit (darwin) configd;
     inherit passthruFun;
   };
@@ -107,10 +167,23 @@ in {
     sourceVersion = {
       major = "3";
       minor = "9";
+      patch = "1";
+      suffix = "";
+    };
+    sha256 = "1zq3k4ymify5ig739zyvx9s2ainvchxb1zpy139z74krr653y74r";
+    inherit (darwin) configd;
+    inherit passthruFun;
+  };
+
+  python310 = callPackage ./cpython {
+    self = python310;
+    sourceVersion = {
+      major = "3";
+      minor = "10";
       patch = "0";
-      suffix = "rc1";
+      suffix = "a3";
     };
-    sha256 = "0w6wvyy9fbvfvrmhvmlb5gq18haagywk9hjkp1knjdarfczag9zv";
+    sha256 = "sha256-sJjJdAdxOUfX7W7VioSGdxlgp2lyMOPZjg42MCd/JYY=";
     inherit (darwin) configd;
     inherit passthruFun;
   };
@@ -118,7 +191,6 @@ in {
   # Minimal versions of Python (built without optional dependencies)
   python3Minimal = (python38.override {
     self = python3Minimal;
-    pythonForBuild = pkgs.buildPackages.python3Minimal;
     # strip down that python version as much as possible
     openssl = null;
     readline = null;
diff --git a/nixpkgs/pkgs/development/interpreters/python/hooks/default.nix b/nixpkgs/pkgs/development/interpreters/python/hooks/default.nix
index d14eb9cbb09d..1a64c79232bc 100644
--- a/nixpkgs/pkgs/development/interpreters/python/hooks/default.nix
+++ b/nixpkgs/pkgs/development/interpreters/python/hooks/default.nix
@@ -1,14 +1,15 @@
 # Hooks for building Python packages.
 { python
 , lib
-, callPackage
 , makeSetupHook
 , disabledIf
 , isPy3k
 , ensureNewerSourcesForZipFilesHook
+, findutils
 }:
 
 let
+  callPackage = python.pythonForBuild.pkgs.callPackage;
   pythonInterpreter = python.pythonForBuild.interpreter;
   pythonSitePackages = python.sitePackages;
   pythonCheckInterpreter = python.interpreter;
@@ -94,7 +95,7 @@ in rec {
     makeSetupHook {
       name = "python-namespaces-hook.sh";
       substitutions = {
-        inherit pythonSitePackages;
+        inherit pythonSitePackages findutils;
       };
     } ./python-namespaces-hook.sh) {};
 
diff --git a/nixpkgs/pkgs/development/interpreters/python/hooks/pip-install-hook.sh b/nixpkgs/pkgs/development/interpreters/python/hooks/pip-install-hook.sh
index 770739b36bde..a4f08b8b14cb 100644
--- a/nixpkgs/pkgs/development/interpreters/python/hooks/pip-install-hook.sh
+++ b/nixpkgs/pkgs/development/interpreters/python/hooks/pip-install-hook.sh
@@ -11,9 +11,7 @@ pipInstallPhase() {
     export PYTHONPATH="$out/@pythonSitePackages@:$PYTHONPATH"
 
     pushd dist || return 1
-    mkdir tmpbuild
-    NIX_PIP_INSTALL_TMPDIR=tmpbuild @pythonInterpreter@ -m pip install ./*.whl --no-index --prefix="$out" --no-cache $pipInstallFlags
-    rm -rf tmpbuild
+    @pythonInterpreter@ -m pip install ./*.whl --no-index --no-warn-script-location --prefix="$out" --no-cache $pipInstallFlags
     popd || return 1
 
     runHook postInstall
diff --git a/nixpkgs/pkgs/development/interpreters/python/hooks/python-namespaces-hook.sh b/nixpkgs/pkgs/development/interpreters/python/hooks/python-namespaces-hook.sh
index 50f21819d176..15d2bd0eb34c 100644
--- a/nixpkgs/pkgs/development/interpreters/python/hooks/python-namespaces-hook.sh
+++ b/nixpkgs/pkgs/development/interpreters/python/hooks/python-namespaces-hook.sh
@@ -17,16 +17,17 @@ pythonNamespacesHook() {
         for pathSegment in ${pathSegments[@]}; do
             constructedPath=${constructedPath}/${pathSegment}
             pathToRemove=${constructedPath}/__init__.py
-            pycachePath=${constructedPath}/__pycache__/__init__*
+            pycachePath=${constructedPath}/__pycache__/
 
+            # remove __init__.py
             if [ -f "$pathToRemove" ]; then
-                echo "Removing $pathToRemove"
-                rm "$pathToRemove"
+                rm -v "$pathToRemove"
             fi
 
-            if [ -f "$pycachePath" ]; then
-                echo "Removing $pycachePath"
-                rm "$pycachePath"
+            # remove __pycache__/ entry, can be interpreter specific. E.g. __init__.cpython-38.pyc
+            # use null characters to perserve potential whitespace in filepath
+            if [ -d "$pycachePath" ]; then
+                @findutils@/bin/find "$pycachePath" -name '__init__*' -exec rm -v "{}" +
             fi
         done
     done
diff --git a/nixpkgs/pkgs/development/interpreters/python/mk-python-derivation.nix b/nixpkgs/pkgs/development/interpreters/python/mk-python-derivation.nix
index c3be76790ebd..670c870f1077 100644
--- a/nixpkgs/pkgs/development/interpreters/python/mk-python-derivation.nix
+++ b/nixpkgs/pkgs/development/interpreters/python/mk-python-derivation.nix
@@ -17,7 +17,6 @@
 , pythonCatchConflictsHook
 , pythonImportsCheckHook
 , pythonNamespacesHook
-, pythonRecompileBytecodeHook
 , pythonRemoveBinBytecodeHook
 , pythonRemoveTestsDirHook
 , setuptoolsBuildHook
@@ -54,7 +53,9 @@
 , disabled ? false
 
 # Raise an error if two packages are installed with the same name
-, catchConflicts ? true
+# TODO: For cross we probably need a different PYTHONPATH, or not
+# add the runtime deps until after buildPhase.
+, catchConflicts ? (python.stdenv.hostPlatform == python.stdenv.buildPlatform)
 
 # Additional arguments to pass to the makeWrapper function, which wraps
 # generated binaries.
@@ -113,7 +114,6 @@ let
       python
       wrapPython
       ensureNewerSourcesForZipFilesHook  # move to wheel installer (pip) or builder (setuptools, flit, ...)?
-      pythonRecompileBytecodeHook  # Remove when solved https://github.com/NixOS/nixpkgs/issues/81441
       pythonRemoveTestsDirHook
     ] ++ lib.optionals catchConflicts [
       setuptools pythonCatchConflictsHook
@@ -167,9 +167,6 @@ let
     # Python packages built through cross-compilation are always for the host platform.
     disallowedReferences = lib.optionals (python.stdenv.hostPlatform != python.stdenv.buildPlatform) [ python.pythonForBuild ];
 
-    # For now, revert recompilation of bytecode.
-    dontUsePythonRecompileBytecode = true;
-
     meta = {
       # default to python's platforms
       platforms = python.meta.platforms;
diff --git a/nixpkgs/pkgs/development/interpreters/python/pypy/default.nix b/nixpkgs/pkgs/development/interpreters/python/pypy/default.nix
index 10073602071a..8feeb3c51bf3 100644
--- a/nixpkgs/pkgs/development/interpreters/python/pypy/default.nix
+++ b/nixpkgs/pkgs/development/interpreters/python/pypy/default.nix
@@ -5,10 +5,16 @@
 , python-setup-hook
 # For the Python package set
 , packageOverrides ? (self: super: {})
+, pkgsBuildBuild
+, pkgsBuildHost
+, pkgsBuildTarget
+, pkgsHostHost
+, pkgsTargetTarget
 , sourceVersion
 , pythonVersion
 , sha256
 , passthruFun
+, pythonAttr ? "pypy${stdenv.lib.substring 0 1 pythonVersion}${stdenv.lib.substring 2 3 pythonVersion}"
 }:
 
 assert zlibSupport -> zlib != null;
@@ -22,9 +28,14 @@ let
     implementation = "pypy";
     libPrefix = "pypy${pythonVersion}";
     executable = "pypy${if isPy3k then "3" else ""}";
-    pythonForBuild = self; # No cross-compiling for now.
     sitePackages = "site-packages";
     hasDistutilsCxxPatch = false;
+
+    pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr};
+    pythonOnBuildForHost = pkgsBuildHost.${pythonAttr};
+    pythonOnBuildForTarget = pkgsBuildTarget.${pythonAttr};
+    pythonOnHostForHost = pkgsHostHost.${pythonAttr};
+    pythonOnTargetForTarget = pkgsTargetTarget.${pythonAttr} or {};
   };
   pname = passthru.executable;
   version = with sourceVersion; "${major}.${minor}.${patch}";
@@ -151,7 +162,7 @@ in with passthru; stdenv.mkDerivation rec {
     homepage = "http://pypy.org/";
     description = "Fast, compliant alternative implementation of the Python language (${pythonVersion})";
     license = licenses.mit;
-    platforms = [ "i686-linux" "x86_64-linux" "x86_64-darwin" ];
+    platforms = [ "aarch64-linux" "i686-linux" "x86_64-linux" "x86_64-darwin" ];
     maintainers = with maintainers; [ andersk ];
   };
 }
diff --git a/nixpkgs/pkgs/development/interpreters/python/tests.nix b/nixpkgs/pkgs/development/interpreters/python/tests.nix
index dcfa41cc308e..a291919b3277 100644
--- a/nixpkgs/pkgs/development/interpreters/python/tests.nix
+++ b/nixpkgs/pkgs/development/interpreters/python/tests.nix
@@ -1,4 +1,5 @@
-{ python
+{ stdenv
+, python
 , runCommand
 , substituteAll
 , lib
@@ -92,4 +93,4 @@ let
 
 
 
-in environmentTests // integrationTests
+in stdenv.lib.optionalAttrs (stdenv.hostPlatform == stdenv.buildPlatform ) (environmentTests // integrationTests)
diff --git a/nixpkgs/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/default.nix b/nixpkgs/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/default.nix
index 05798cbaf1b8..572cbdccbfb2 100644
--- a/nixpkgs/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/default.nix
+++ b/nixpkgs/pkgs/development/interpreters/python/tests/test_nix_pythonprefix/default.nix
@@ -4,7 +4,7 @@ let
 
   python = let
     packageOverrides = self: super: {
-      typeddep = super.callPackage ./typeddep {};
+      typeddep = self.callPackage ./typeddep {};
     };
   in interpreter.override {inherit packageOverrides; self = python;};
 
diff --git a/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/default.nix b/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/default.nix
index 762ca2bdd34b..81975bc5250e 100644
--- a/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/default.nix
+++ b/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/default.nix
@@ -9,4 +9,4 @@ runCommand "update-python-libraries" {
   cp ${./update-python-libraries.py} $out
   patchShebangs $out
   substituteInPlace $out --replace 'GIT = "git"' 'GIT = "${git}/bin/git"'
-''
\ No newline at end of file
+''
diff --git a/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py b/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py
index b568ee6751d7..9054195ab7e6 100755
--- a/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py
+++ b/nixpkgs/pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py
@@ -12,11 +12,10 @@ to update all non-pinned libraries in that folder.
 """
 
 import argparse
-import logging
 import os
+import pathlib
 import re
 import requests
-import toolz
 from concurrent.futures import ThreadPoolExecutor as Pool
 from packaging.version import Version as _Version
 from packaging.version import InvalidVersion
@@ -34,6 +33,8 @@ PRERELEASES = False
 
 GIT = "git"
 
+NIXPGKS_ROOT = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode('utf-8').strip()
+
 import logging
 logging.basicConfig(level=logging.INFO)
 
@@ -101,6 +102,7 @@ def _replace_value(attribute, value, text):
     new_text = text.replace(old_line, new_line)
     return new_text
 
+
 def _fetch_page(url):
     r = requests.get(url)
     if r.status_code == requests.codes.ok:
@@ -109,6 +111,19 @@ def _fetch_page(url):
         raise ValueError("request for {} failed".format(url))
 
 
+def _fetch_github(url):
+    headers = {}
+    token = os.environ.get('GITHUB_API_TOKEN')
+    if token:
+        headers["Authorization"] = f"token {token}"
+    r = requests.get(url, headers=headers)
+
+    if r.status_code == requests.codes.ok:
+        return r.json()
+    else:
+        raise ValueError("request for {} failed".format(url))
+
+
 SEMVER = {
     'major' : 0,
     'minor' : 1,
@@ -167,11 +182,45 @@ def _get_latest_version_pypi(package, extension, current_version, target):
             break
     else:
         sha256 = None
-    return version, sha256
+    return version, sha256, None
 
 
 def _get_latest_version_github(package, extension, current_version, target):
-    raise ValueError("updating from GitHub is not yet supported.")
+    def strip_prefix(tag):
+        return re.sub("^[^0-9]*", "", tag)
+
+    def get_prefix(string):
+        matches = re.findall(r"^([^0-9]*)", string)
+        return next(iter(matches), "")
+
+    # when invoked as an updateScript, UPDATE_NIX_ATTR_PATH will be set
+    # this allows us to work with packages which live outside of python-modules
+    attr_path = os.environ.get("UPDATE_NIX_ATTR_PATH", f"python3Packages.{package}")
+    try:
+        homepage = subprocess.check_output(
+            ["nix", "eval", "-f", f"{NIXPGKS_ROOT}/default.nix", "--raw", f"{attr_path}.src.meta.homepage"])\
+            .decode('utf-8')
+    except Exception as e:
+        raise ValueError(f"Unable to determine homepage: {e}")
+    owner_repo = homepage[len("https://github.com/"):]  # remove prefix
+    owner, repo = owner_repo.split("/")
+
+    url = f"https://api.github.com/repos/{owner}/{repo}/releases"
+    all_releases = _fetch_github(url)
+    releases = list(filter(lambda x: not x['prerelease'], all_releases))
+
+    if len(releases) == 0:
+        raise ValueError(f"{homepage} does not contain any stable releases")
+
+    versions = map(lambda x: strip_prefix(x['tag_name']), releases)
+    version = _determine_latest_version(current_version, target, versions)
+
+    release = next(filter(lambda x: strip_prefix(x['tag_name']) == version, releases))
+    prefix = get_prefix(release['tag_name'])
+    sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", "--unpack", f"{release['tarball_url']}"], stderr=subprocess.DEVNULL)\
+        .decode('utf-8').strip()
+
+    return version, sha256, prefix
 
 
 FETCHERS = {
@@ -240,7 +289,9 @@ def _determine_extension(text, fetcher):
             raise ValueError('url does not point to PyPI.')
 
     elif fetcher == 'fetchFromGitHub':
-        raise ValueError('updating from GitHub is not yet implemented.')
+        if "fetchSubmodules" in text:
+            raise ValueError("fetchFromGitHub fetcher doesn't support submodules")
+        extension = "tar.gz"
 
     return extension
 
@@ -262,7 +313,7 @@ def _update_package(path, target):
 
     extension = _determine_extension(text, fetcher)
 
-    new_version, new_sha256 = FETCHERS[fetcher](pname, extension, version, target)
+    new_version, new_sha256, prefix = FETCHERS[fetcher](pname, extension, version, target)
 
     if new_version == version:
         logging.info("Path {}: no update available for {}.".format(path, pname))
@@ -274,6 +325,10 @@ def _update_package(path, target):
 
     text = _replace_value('version', new_version, text)
     text = _replace_value('sha256', new_sha256, text)
+    if fetcher == 'fetchFromGitHub':
+        text = _replace_value('rev', f"{prefix}${{version}}", text)
+        # incase there's no prefix, just rewrite without interpolation
+        text = text.replace('"${version}";', 'version;')
 
     with open(path, 'w') as f:
         f.write(text)
@@ -333,7 +388,11 @@ def _commit(path, pname, old_version, new_version, pkgs_prefix="python: ", **kwa
 
 def main():
 
-    parser = argparse.ArgumentParser()
+    epilog = """
+environment variables:
+  GITHUB_API_TOKEN\tGitHub API token used when updating github packages
+    """
+    parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog)
     parser.add_argument('package', type=str, nargs='+')
     parser.add_argument('--target', type=str, choices=SEMVER.keys(), default='major')
     parser.add_argument('--commit', action='store_true', help='Create a commit for each package update')