diff options
Diffstat (limited to 'nixpkgs/pkgs/development/python-modules/cython')
3 files changed, 448 insertions, 0 deletions
diff --git a/nixpkgs/pkgs/development/python-modules/cython/default.nix b/nixpkgs/pkgs/development/python-modules/cython/default.nix new file mode 100644 index 000000000000..72ba4a68f038 --- /dev/null +++ b/nixpkgs/pkgs/development/python-modules/cython/default.nix @@ -0,0 +1,91 @@ +{ lib +, stdenv +, buildPythonPackage +, fetchPypi +, fetchpatch +, setuptools +, python +, pkg-config +, gdb +, numpy +, ncurses +}: + +let + excludedTests = [ "reimport_from_subinterpreter" ] + # cython's testsuite is not working very well with libc++ + # We are however optimistic about things outside of testsuite still working + ++ lib.optionals (stdenv.cc.isClang or false) [ "cpdef_extern_func" "libcpp_algo" ] + # Some tests in the test suite isn't working on aarch64. Disable them for + # now until upstream finds a workaround. + # Upstream issue here: https://github.com/cython/cython/issues/2308 + ++ lib.optionals stdenv.isAarch64 [ "numpy_memoryview" ] + ++ lib.optionals stdenv.isi686 [ "future_division" "overflow_check_longlong" ] + ; + +in buildPythonPackage rec { + pname = "cython"; + version = "0.29.36"; + pyproject = true; + + src = fetchPypi { + pname = "Cython"; + inherit version; + hash = "sha256-QcDP0tdU44PJ7rle/8mqSrhH0Ml0cHfd18Dctow7wB8="; + }; + + nativeBuildInputs = [ + pkg-config + setuptools + ]; + + nativeCheckInputs = [ + gdb numpy ncurses + ]; + + LC_ALL = "en_US.UTF-8"; + + patches = [ + # backport Cython 3.0 trashcan support (https://github.com/cython/cython/pull/2842) to 0.X series. + # it does not affect Python code unless the code explicitly uses the feature. + # trashcan support is needed to avoid stack overflows during object deallocation in sage (https://trac.sagemath.org/ticket/27267) + ./trashcan.patch + # The above commit introduces custom trashcan macros, as well as + # compiler changes to use them in Cython-emitted code. The latter + # change is still useful, but the former has been upstreamed as of + # Python 3.8, and the patch below makes Cython use the upstream + # trashcan macros whenever available. This is needed for Python + # 3.11 support, because the API used in Cython's implementation + # changed: https://github.com/cython/cython/pull/4475 + (fetchpatch { + name = "disable-trashcan.patch"; + url = "https://github.com/cython/cython/commit/e337825cdcf5e94d38ba06a0cb0188e99ce0cc92.patch"; + hash = "sha256-q0f63eetKrDpmP5Z4v8EuGxg26heSyp/62OYqhRoSso="; + }) + ]; + + checkPhase = '' + export HOME="$NIX_BUILD_TOP" + ${python.interpreter} runtests.py -j$NIX_BUILD_CORES \ + --no-code-style \ + ${lib.optionalString (builtins.length excludedTests != 0) + ''--exclude="(${builtins.concatStringsSep "|" excludedTests})"''} + ''; + + # https://github.com/cython/cython/issues/2785 + # Temporary solution + doCheck = false; + # doCheck = !stdenv.isDarwin; + + # force regeneration of generated code in source distributions + # https://github.com/cython/cython/issues/5089 + setupHook = ./setup-hook.sh; + + meta = { + changelog = "https://github.com/cython/cython/blob/${version}/CHANGES.rst"; + description = "An optimising static compiler for both the Python programming language and the extended Cython programming language"; + homepage = "https://cython.org"; + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ fridh ]; + }; +} diff --git a/nixpkgs/pkgs/development/python-modules/cython/setup-hook.sh b/nixpkgs/pkgs/development/python-modules/cython/setup-hook.sh new file mode 100644 index 000000000000..c26330a53644 --- /dev/null +++ b/nixpkgs/pkgs/development/python-modules/cython/setup-hook.sh @@ -0,0 +1,3 @@ +if [ -z "${dontForceRegenCython-}"]; then + export CYTHON_FORCE_REGEN=1 +fi diff --git a/nixpkgs/pkgs/development/python-modules/cython/trashcan.patch b/nixpkgs/pkgs/development/python-modules/cython/trashcan.patch new file mode 100644 index 000000000000..398422e76f39 --- /dev/null +++ b/nixpkgs/pkgs/development/python-modules/cython/trashcan.patch @@ -0,0 +1,354 @@ +From 1b77e35d848340f2c5f4c9b82965c25a0572d48f Mon Sep 17 00:00:00 2001 +From: Jeroen Demeyer <J.Demeyer@UGent.be> +Date: Thu, 14 Feb 2019 10:02:41 +0100 +Subject: [PATCH] @cython.trashcan directive to enable the Python trashcan for + deallocations + +--- + Cython/Compiler/ModuleNode.py | 10 +++ + Cython/Compiler/Options.py | 2 + + Cython/Compiler/PyrexTypes.py | 8 +- + Cython/Compiler/Symtab.py | 18 +++- + Cython/Utility/ExtensionTypes.c | 43 ++++++++++ + tests/run/trashcan.pyx | 148 ++++++++++++++++++++++++++++++++ + 6 files changed, 227 insertions(+), 2 deletions(-) + create mode 100644 tests/run/trashcan.pyx + +diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py +index 56845330d..3a3e8a956 100644 +--- a/Cython/Compiler/ModuleNode.py ++++ b/Cython/Compiler/ModuleNode.py +@@ -1443,6 +1443,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): + + is_final_type = scope.parent_type.is_final_type + needs_gc = scope.needs_gc() ++ needs_trashcan = scope.needs_trashcan() + + weakref_slot = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None + if weakref_slot not in scope.var_entries: +@@ -1481,6 +1482,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): + # running this destructor. + code.putln("PyObject_GC_UnTrack(o);") + ++ if needs_trashcan: ++ code.globalstate.use_utility_code( ++ UtilityCode.load_cached("PyTrashcan", "ExtensionTypes.c")) ++ code.putln("__Pyx_TRASHCAN_BEGIN(o, %s)" % slot_func_cname) ++ + # call the user's __dealloc__ + self.generate_usr_dealloc_call(scope, code) + +@@ -1554,6 +1560,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): + code.putln("(*Py_TYPE(o)->tp_free)(o);") + if freelist_size: + code.putln("}") ++ ++ if needs_trashcan: ++ code.putln("__Pyx_TRASHCAN_END") ++ + code.putln( + "}") + +diff --git a/Cython/Compiler/Options.py b/Cython/Compiler/Options.py +index d03119fca..05a728135 100644 +--- a/Cython/Compiler/Options.py ++++ b/Cython/Compiler/Options.py +@@ -319,6 +319,7 @@ directive_types = { + 'freelist': int, + 'c_string_type': one_of('bytes', 'bytearray', 'str', 'unicode'), + 'c_string_encoding': normalise_encoding_name, ++ 'trashcan': bool, + 'cpow': bool + } + +@@ -362,6 +363,7 @@ directive_scopes = { # defaults to available everywhere + 'np_pythran': ('module',), + 'fast_gil': ('module',), + 'iterable_coroutine': ('module', 'function'), ++ 'trashcan' : ('cclass',), + } + + +diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py +index c309bd04b..9231130b5 100644 +--- a/Cython/Compiler/PyrexTypes.py ++++ b/Cython/Compiler/PyrexTypes.py +@@ -1129,6 +1129,7 @@ class PyObjectType(PyrexType): + is_extern = False + is_subclassed = False + is_gc_simple = False ++ builtin_trashcan = False # builtin type using trashcan + + def __str__(self): + return "Python object" +@@ -1183,10 +1184,14 @@ class PyObjectType(PyrexType): + + + builtin_types_that_cannot_create_refcycles = set([ +- 'bool', 'int', 'long', 'float', 'complex', ++ 'object', 'bool', 'int', 'long', 'float', 'complex', + 'bytearray', 'bytes', 'unicode', 'str', 'basestring' + ]) + ++builtin_types_with_trashcan = set([ ++ 'dict', 'list', 'set', 'frozenset', 'tuple', 'type', ++]) ++ + + class BuiltinObjectType(PyObjectType): + # objstruct_cname string Name of PyObject struct +@@ -1211,6 +1216,7 @@ class BuiltinObjectType(PyObjectType): + self.typeptr_cname = "(&%s)" % cname + self.objstruct_cname = objstruct_cname + self.is_gc_simple = name in builtin_types_that_cannot_create_refcycles ++ self.builtin_trashcan = name in builtin_types_with_trashcan + if name == 'type': + # Special case the type type, as many C API calls (and other + # libraries) actually expect a PyTypeObject* for type arguments. +diff --git a/Cython/Compiler/Symtab.py b/Cython/Compiler/Symtab.py +index 7361a55ae..f0c311ba6 100644 +--- a/Cython/Compiler/Symtab.py ++++ b/Cython/Compiler/Symtab.py +@@ -2043,7 +2043,7 @@ class PyClassScope(ClassScope): + class CClassScope(ClassScope): + # Namespace of an extension type. + # +- # parent_type CClassType ++ # parent_type PyExtensionType + # #typeobj_cname string or None + # #objstruct_cname string + # method_table_cname string +@@ -2087,6 +2087,22 @@ class CClassScope(ClassScope): + return not self.parent_type.is_gc_simple + return False + ++ def needs_trashcan(self): ++ # If the trashcan directive is explicitly set to False, ++ # unconditionally disable the trashcan. ++ directive = self.directives.get('trashcan') ++ if directive is False: ++ return False ++ # If the directive is set to True and the class has Python-valued ++ # C attributes, then it should use the trashcan in tp_dealloc. ++ if directive and self.has_cyclic_pyobject_attrs: ++ return True ++ # Use the trashcan if the base class uses it ++ base_type = self.parent_type.base_type ++ if base_type and base_type.scope is not None: ++ return base_type.scope.needs_trashcan() ++ return self.parent_type.builtin_trashcan ++ + def needs_tp_clear(self): + """ + Do we need to generate an implementation for the tp_clear slot? Can +diff --git a/Cython/Utility/ExtensionTypes.c b/Cython/Utility/ExtensionTypes.c +index dc187ab49..f359165df 100644 +--- a/Cython/Utility/ExtensionTypes.c ++++ b/Cython/Utility/ExtensionTypes.c +@@ -119,6 +119,49 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { + return r; + } + ++/////////////// PyTrashcan.proto /////////////// ++ ++// These macros are taken from https://github.com/python/cpython/pull/11841 ++// Unlike the Py_TRASHCAN_SAFE_BEGIN/Py_TRASHCAN_SAFE_END macros, they ++// allow dealing correctly with subclasses. ++ ++// This requires CPython version >= 2.7.4 ++// (or >= 3.2.4 but we don't support such old Python 3 versions anyway) ++#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070400 ++#define __Pyx_TRASHCAN_BEGIN_CONDITION(op, cond) \ ++ do { \ ++ PyThreadState *_tstate = NULL; \ ++ // If "cond" is false, then _tstate remains NULL and the deallocator ++ // is run normally without involving the trashcan ++ if (cond) { \ ++ _tstate = PyThreadState_GET(); \ ++ if (_tstate->trash_delete_nesting >= PyTrash_UNWIND_LEVEL) { \ ++ // Store the object (to be deallocated later) and jump past ++ // Py_TRASHCAN_END, skipping the body of the deallocator ++ _PyTrash_thread_deposit_object((PyObject*)(op)); \ ++ break; \ ++ } \ ++ ++_tstate->trash_delete_nesting; \ ++ } ++ // The body of the deallocator is here. ++#define __Pyx_TRASHCAN_END \ ++ if (_tstate) { \ ++ --_tstate->trash_delete_nesting; \ ++ if (_tstate->trash_delete_later && _tstate->trash_delete_nesting <= 0) \ ++ _PyTrash_thread_destroy_chain(); \ ++ } \ ++ } while (0); ++ ++#define __Pyx_TRASHCAN_BEGIN(op, dealloc) __Pyx_TRASHCAN_BEGIN_CONDITION(op, \ ++ Py_TYPE(op)->tp_dealloc == (destructor)(dealloc)) ++ ++#else ++// The trashcan is a no-op on other Python implementations ++// or old CPython versions ++#define __Pyx_TRASHCAN_BEGIN(op, dealloc) ++#define __Pyx_TRASHCAN_END ++#endif ++ + /////////////// CallNextTpDealloc.proto /////////////// + + static void __Pyx_call_next_tp_dealloc(PyObject* obj, destructor current_tp_dealloc); +diff --git a/tests/run/trashcan.pyx b/tests/run/trashcan.pyx +new file mode 100644 +index 000000000..93a501ff8 +--- /dev/null ++++ b/tests/run/trashcan.pyx +@@ -0,0 +1,148 @@ ++# mode: run ++ ++cimport cython ++ ++ ++# Count number of times an object was deallocated twice. This should remain 0. ++cdef int double_deallocations = 0 ++def assert_no_double_deallocations(): ++ global double_deallocations ++ err = double_deallocations ++ double_deallocations = 0 ++ assert not err ++ ++ ++# Compute x = f(f(f(...(None)...))) nested n times and throw away the result. ++# The real test happens when exiting this function: then a big recursive ++# deallocation of x happens. We are testing two things in the tests below: ++# that Python does not crash and that no double deallocation happens. ++# See also https://github.com/python/cpython/pull/11841 ++def recursion_test(f, int n=2**20): ++ x = None ++ cdef int i ++ for i in range(n): ++ x = f(x) ++ ++ ++@cython.trashcan(True) ++cdef class Recurse: ++ """ ++ >>> recursion_test(Recurse) ++ >>> assert_no_double_deallocations() ++ """ ++ cdef public attr ++ cdef int deallocated ++ ++ def __init__(self, x): ++ self.attr = x ++ ++ def __dealloc__(self): ++ # Check that we're not being deallocated twice ++ global double_deallocations ++ double_deallocations += self.deallocated ++ self.deallocated = 1 ++ ++ ++cdef class RecurseSub(Recurse): ++ """ ++ >>> recursion_test(RecurseSub) ++ >>> assert_no_double_deallocations() ++ """ ++ cdef int subdeallocated ++ ++ def __dealloc__(self): ++ # Check that we're not being deallocated twice ++ global double_deallocations ++ double_deallocations += self.subdeallocated ++ self.subdeallocated = 1 ++ ++ ++@cython.freelist(4) ++@cython.trashcan(True) ++cdef class RecurseFreelist: ++ """ ++ >>> recursion_test(RecurseFreelist) ++ >>> recursion_test(RecurseFreelist, 1000) ++ >>> assert_no_double_deallocations() ++ """ ++ cdef public attr ++ cdef int deallocated ++ ++ def __init__(self, x): ++ self.attr = x ++ ++ def __dealloc__(self): ++ # Check that we're not being deallocated twice ++ global double_deallocations ++ double_deallocations += self.deallocated ++ self.deallocated = 1 ++ ++ ++# Subclass of list => uses trashcan by default ++# As long as https://github.com/python/cpython/pull/11841 is not fixed, ++# this does lead to double deallocations, so we skip that check. ++cdef class RecurseList(list): ++ """ ++ >>> RecurseList(42) ++ [42] ++ >>> recursion_test(RecurseList) ++ """ ++ def __init__(self, x): ++ super().__init__((x,)) ++ ++ ++# Some tests where the trashcan is NOT used. When the trashcan is not used ++# in a big recursive deallocation, the __dealloc__s of the base classs are ++# only run after the __dealloc__s of the subclasses. ++# We use this to detect trashcan usage. ++cdef int base_deallocated = 0 ++cdef int trashcan_used = 0 ++def assert_no_trashcan_used(): ++ global base_deallocated, trashcan_used ++ err = trashcan_used ++ trashcan_used = base_deallocated = 0 ++ assert not err ++ ++ ++cdef class Base: ++ def __dealloc__(self): ++ global base_deallocated ++ base_deallocated = 1 ++ ++ ++# Trashcan disabled by default ++cdef class Sub1(Base): ++ """ ++ >>> recursion_test(Sub1, 100) ++ >>> assert_no_trashcan_used() ++ """ ++ cdef public attr ++ ++ def __init__(self, x): ++ self.attr = x ++ ++ def __dealloc__(self): ++ global base_deallocated, trashcan_used ++ trashcan_used += base_deallocated ++ ++ ++@cython.trashcan(True) ++cdef class Middle(Base): ++ cdef public foo ++ ++ ++# Trashcan disabled explicitly ++@cython.trashcan(False) ++cdef class Sub2(Middle): ++ """ ++ >>> recursion_test(Sub2, 1000) ++ >>> assert_no_trashcan_used() ++ """ ++ cdef public attr ++ ++ def __init__(self, x): ++ self.attr = x ++ ++ def __dealloc__(self): ++ global base_deallocated, trashcan_used ++ trashcan_used += base_deallocated +-- +2.39.0 + |