Merge pull request #36899 from aborsu/spacy

Spacy

6 files changed, 130 insertions, 73 deletions

diff --git a/pkgs/development/python-modules/ftfy/default.nix b/pkgs/development/python-modules/ftfy/default.nix
index d88c894009c1..ca1c9a09e4eb 100644
--- a/pkgs/development/python-modules/ftfy/default.nix
+++ b/pkgs/development/python-modules/ftfy/default.nix
@@ -7,18 +7,23 @@
 , python
 , isPy3k
 }:
+
 buildPythonPackage rec {
-  name = "${pname}-${version}";
   pname = "ftfy";
-  # latest is 5.1.1, buy spaCy requires 4.4.3
-  version = "5.3.0";
+
+  version = "4.4.3";
+  # ftfy v5 only supports python3. Since at the moment the only
+  # packages that use ftfy are spacy and textacy which both support
+  # python 2 and 3, they have pinned ftfy to the v4 branch.
+  # I propose to stick to v4 until another package requires v5.
+  # At that point we can make a ftfy_v4 package.
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "0ba702d5138f9b35df32b55920c9466208608108f1f3d5de1a68c17e3d68cb7f";
+    sha256 = "152xdb56rhs1q4r0ck1n557sbphw7zq18r75a7kkd159ckdnc01w";
   };
 
-  propagatedBuildInputs = [ html5lib wcwidth];
+  propagatedBuildInputs = [ html5lib wcwidth ];
 
   checkInputs = [
     nose
@@ -32,13 +37,10 @@ buildPythonPackage rec {
   # FileNotFoundError: [Errno 2] No such file or directory: 'ftfy'
   doCheck = false;
 
-  # "this version of ftfy is no longer written for Python 2"
-  disabled = !isPy3k;
-
   meta = with stdenv.lib; {
     description = "Given Unicode text, make its representation consistent and possibly less broken.";
     homepage = https://github.com/LuminosoInsight/python-ftfy/tree/master/tests;
     license = licenses.mit;
-    maintainers = with maintainers; [ sdll ];
-    };
+    maintainers = with maintainers; [ sdll aborsu ];
+  };
 }
diff --git a/pkgs/development/python-modules/msgpack-numpy/default.nix b/pkgs/development/python-modules/msgpack-numpy/default.nix
new file mode 100644
index 000000000000..0244b03c4842
--- /dev/null
+++ b/pkgs/development/python-modules/msgpack-numpy/default.nix
@@ -0,0 +1,38 @@
+{ stdenv
+, buildPythonPackage
+, fetchPypi
+, cython
+, msgpack-python
+, numpy
+, python
+}:
+
+buildPythonPackage rec {
+  pname = "msgpack-numpy";
+  version = "0.4.1";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1czf125nixzwskiqiw0145kfj15030sp334cb89gp5w4rz3h7img";
+  };
+
+  buildInputs = [
+    cython
+  ];
+
+  propagatedBuildInputs = [
+   msgpack-python
+   numpy
+  ];
+
+  checkPhase = ''
+    ${python.interpreter} msgpack_numpy.py
+  '';
+
+  meta = with stdenv.lib; {
+    description = "Practical Machine Learning for NLP in Python";
+    homepage = https://github.com/lebedov/msgpack-numpy;
+    license = licenses.bsd3;
+    maintainers = with maintainers; [ aborsu ];
+  };
+}
diff --git a/pkgs/development/python-modules/murmurhash/default.nix b/pkgs/development/python-modules/murmurhash/default.nix
index 023c94cdc8ea..f689bc2ff49d 100644
--- a/pkgs/development/python-modules/murmurhash/default.nix
+++ b/pkgs/development/python-modules/murmurhash/default.nix
@@ -1,35 +1,34 @@
 { stdenv
 , buildPythonPackage
-, fetchFromGitHub
+, fetchPypi
 , cython
 , python
 }:
 
 buildPythonPackage rec {
   pname = "murmurhash";
-  version = "0.26.4";
-  name = pname + "-" + version;
+  version = "0.28.0";
 
-  src = fetchFromGitHub {
-    owner = "explosion";
-    repo = "murmurhash";
-    rev = "0.26.4";
-    sha256 = "0n2j0glhlv2yh3fjgbg4d79j1c1fpchgjd4vnpw908l9mzchhmdv";
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "16id8jppw8r54wisrlaaiprcszzb7d7lbpnskqn38s8i7vnkf4b5";
   };
 
   buildInputs = [
    cython
   ];
 
+  # No test
+  doCheck = false;
+
   checkPhase = ''
-    cd murmurhash/tests
-    ${python.interpreter} -m unittest discover -p "*test*"
+    pytest murmurhash
   '';
 
   meta = with stdenv.lib; {
     description = "Cython bindings for MurmurHash2";
     homepage = https://github.com/explosion/murmurhash;
     license = licenses.mit;
-    maintainers = with maintainers; [ sdll ];
-    };
+    maintainers = with maintainers; [ aborsu sdll ];
+  };
 }
diff --git a/pkgs/development/python-modules/spacy/default.nix b/pkgs/development/python-modules/spacy/default.nix
index 6c7c3c8400c1..b94374aeda86 100644
--- a/pkgs/development/python-modules/spacy/default.nix
+++ b/pkgs/development/python-modules/spacy/default.nix
@@ -3,12 +3,11 @@
 , buildPythonPackage
 , python
 , fetchPypi
-, fetchFromGitHub
+, html5lib
 , pytest
 , cython
-, cymem
 , preshed
-, pathlib2
+, ftfy
 , numpy
 , murmurhash
 , plac
@@ -16,7 +15,6 @@
 , ujson
 , dill
 , requests
-, ftfy
 , thinc
 , pip
 , regex
@@ -24,32 +22,49 @@
 
 buildPythonPackage rec {
   pname = "spacy";
-  version = "1.8.2";
+  version = "2.0.9";
 
-  src = fetchFromGitHub {
-    owner = "explosion";
-    repo = "spaCy";
-    rev = "v${version}";
-    sha256 = "0v3bmmar31a6968y4wl0lmgnc3829l2mnwd8s959m4pqw1y1w648";
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1ihkhflhyz67bp73kfjqfrbcgdxi2msz5asbrh0pkk590c4vmms5";
   };
 
+  prePatch = ''
+    substituteInPlace setup.py --replace \
+      "'html5lib==1.0b8'," \
+      "'html5lib',"
+
+    substituteInPlace setup.py --replace \
+      "'regex==2017.4.5'," \
+      "'regex',"
+
+    substituteInPlace setup.py --replace \
+      "'ftfy==2017.4.5'," \
+      "'ftfy',"
+
+    substituteInPlace setup.py --replace \
+      "'pathlib'," \
+      "\"pathlib; python_version<'3.4'\","
+  '';
+
   propagatedBuildInputs = [
    cython
-   cymem
-   pathlib2
-   preshed
-   numpy
+   dill
+   html5lib
    murmurhash
+   numpy
    plac
+   preshed
+   regex
+   requests
    six
+   thinc
    ujson
-   dill
-   requests
-   regex
    ftfy
-   thinc
-   pytest
-   pip
+  ];
+
+  checkInputs = [
+    pytest
   ];
 
   doCheck = false;
diff --git a/pkgs/development/python-modules/thinc/default.nix b/pkgs/development/python-modules/thinc/default.nix
index 15794caf58b9..2c6df953e169 100644
--- a/pkgs/development/python-modules/thinc/default.nix
+++ b/pkgs/development/python-modules/thinc/default.nix
@@ -1,15 +1,19 @@
 { stdenv
+, lib
 , pkgs
 , buildPythonPackage
 , fetchPypi
-, fetchFromGitHub
+, pythonOlder
 , pytest
 , cython
 , cymem
+, msgpack-numpy
+, msgpack-python
 , preshed
 , numpy
 , python
 , murmurhash
+, pathlib
 , hypothesis
 , tqdm
 , cytoolz
@@ -21,36 +25,21 @@
 , dill
 }:
 
-let
-  enableDebugging = true;
-
-  pathlibLocked = buildPythonPackage rec {
-    name = "${pname}-${version}";
-    pname = "pathlib";
-    version = "1.0.1";
-
-    src = fetchPypi {
-      inherit pname version;
-      sha256 = "17zajiw4mjbkkv6ahp3xf025qglkj0805m9s41c45zryzj6p2h39";
-    };
-
-    doCheck = false; # fails to import support from test
-  };
-in buildPythonPackage rec {
+buildPythonPackage rec {
   pname = "thinc";
-  version = "6.5.1";
+  version = "6.10.2";
   name = pname + "-" + version;
 
-  src = fetchFromGitHub {
-    owner = "explosion";
-    repo = "thinc";
-    rev = "v${version}";
-    sha256 = "008kmjsvanh6qgnpvsn3qacfcyprxirxbw4yfd8flyg7mxw793ws";
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "0xia81wvfrhyriywab184s49g8rpl42vcf5fy3x6xxw50a2yn7cs";
   };
 
   propagatedBuildInputs = [
    cython
    cymem
+   msgpack-numpy
+   msgpack-python
    preshed
    numpy
    murmurhash
@@ -64,22 +53,34 @@ in buildPythonPackage rec {
    termcolor
    wrapt
    dill
-   pathlibLocked
+  ] ++ lib.optional (pythonOlder "3.4") pathlib;
+
+
+  checkInputs = [
+    pytest
   ];
 
+  prePatch = ''
+    substituteInPlace setup.py --replace \
+      "'pathlib>=1.0.0,<2.0.0'," \
+      "\"pathlib>=1.0.0,<2.0.0; python_version<'3.4'\","
+
+    substituteInPlace setup.py --replace \
+      "'cytoolz>=0.8,<0.9'," \
+      "'cytoolz>=0.8',"
+  '';
+
+  # Cannot find cython modules.
   doCheck = false;
 
-  # fails to import some modules
-  # checkPhase = ''
-  #   ${python.interpreter} -m pytest thinc/tests
-  #   # cd thinc/tests
-  #   # ${python.interpreter} -m unittest discover -p "*test*"
-  # '';
+  checkPhase = ''
+    pytest thinc/tests
+  '';
 
   meta = with stdenv.lib; {
     description = "Practical Machine Learning for NLP in Python";
     homepage = https://github.com/explosion/thinc;
     license = licenses.mit;
-    maintainers = with maintainers; [ sdll ];
+    maintainers = with maintainers; [ aborsu sdll ];
     };
 }
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 009a1da60cd0..373c3a27de2c 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -10138,6 +10138,8 @@ in {
 
   msgpack = callPackage ../development/python-modules/msgpack {};
 
+  msgpack-numpy = callPackage ../development/python-modules/msgpack-numpy {};
+
   msgpack-python = self.msgpack.overridePythonAttrs {
     pname = "msgpack-python";
     postPatch = ''