Merge commit '891f607d5301d6730cb1f9dcf3618bcb1ab7f10e'

1 files changed, 16 insertions, 5 deletions

diff --git a/nixpkgs/pkgs/development/python-modules/tokenizers/default.nix b/nixpkgs/pkgs/development/python-modules/tokenizers/default.nix
index 4e5aacd6c7ac..ec435a0ac359 100644
--- a/nixpkgs/pkgs/development/python-modules/tokenizers/default.nix
+++ b/nixpkgs/pkgs/development/python-modules/tokenizers/default.nix
@@ -1,4 +1,4 @@
-{ stdenv
+{ lib, stdenv
 , rustPlatform
 , fetchFromGitHub
 , fetchurl
@@ -7,6 +7,7 @@
 , wheel
 , numpy
 , python
+, datasets
 , pytestCheckHook
 , requests
 }:
@@ -50,16 +51,16 @@ let
   };
 in rustPlatform.buildRustPackage rec {
   pname = "tokenizers";
-  version = "0.9.4";
+  version = "0.10.0";
 
   src = fetchFromGitHub {
     owner = "huggingface";
     repo = pname;
     rev = "python-v${version}";
-    hash = "sha256-JXoH9yfhMIFg5qDY5zrF6iWb7XKugjMfk1NxSizfaWg=";
+    hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw=";
   };
 
-  cargoSha256 = "sha256-u9qitrOxJSABs0VjwHUZgmw7VTQXNbp6l8fKKE/RQ7M=";
+  cargoSha256 = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
 
   sourceRoot = "source/bindings/python";
 
@@ -75,6 +76,7 @@ in rustPlatform.buildRustPackage rec {
   ];
 
   installCheckInputs = [
+    datasets
     pytestCheckHook
     requests
   ];
@@ -105,7 +107,16 @@ in rustPlatform.buildRustPackage rec {
     pipInstallPhase
   '';
 
-  meta = with stdenv.lib; {
+  preCheck = ''
+    HOME=$TMPDIR
+  '';
+
+  disabledTests = [
+    # Downloads data using the datasets module.
+    "TestTrainFromIterators"
+  ];
+
+  meta = with lib; {
     homepage = "https://github.com/huggingface/tokenizers";
     description = "Fast State-of-the-Art Tokenizers optimized for Research and Production";
     license = licenses.asl20;