1 files changed, 15 insertions, 22 deletions
diff --git a/nixpkgs/pkgs/development/python-modules/tokenizers/default.nix b/nixpkgs/pkgs/development/python-modules/tokenizers/default.nix
index cf122613f633..348302a59c55 100644
--- a/nixpkgs/pkgs/development/python-modules/tokenizers/default.nix
+++ b/nixpkgs/pkgs/development/python-modules/tokenizers/default.nix
@@ -1,12 +1,10 @@
 { lib
-, rustPlatform
 , fetchFromGitHub
 , fetchurl
-, pipInstallHook
+, buildPythonPackage
+, rustPlatform
 , setuptools-rust
-, wheel
 , numpy
-, python
 , datasets
 , pytestCheckHook
 , requests
@@ -49,30 +47,33 @@ let
     url = "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-merges.txt";
     sha256 = "09a754pm4djjglv3x5pkgwd6f79i2rq8ydg0f7c3q1wmwqdbba8f";
   };
-in rustPlatform.buildRustPackage rec {
+in buildPythonPackage rec {
   pname = "tokenizers";
-  version = "0.10.0";
+  version = "0.10.1";
 
   src = fetchFromGitHub {
     owner = "huggingface";
     repo = pname;
     rev = "python-v${version}";
-    hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw=";
+    hash = "sha256-N/dKjQwHKmJnB76q8ISQ3cjuW0Z4GqGavnFFx/w9JRQ=";
   };
 
-  cargoSha256 = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
+  cargoDeps = rustPlatform.fetchCargoTarball {
+    inherit src sourceRoot;
+    name = "${pname}-${version}";
+    hash = "sha256-3ICSjtiRfLOj+PXu6mcuDoAtod5uXAcabYWTLxEgI18=";
+  };
 
   sourceRoot = "source/bindings/python";
 
-  nativeBuildInputs = [
-    pipInstallHook
-    setuptools-rust
-    wheel
-  ];
+  nativeBuildInputs = [ setuptools-rust ] ++ (with rustPlatform; [
+    cargoSetupHook
+    rust.cargo
+    rust.rustc
+  ]);
 
   propagatedBuildInputs = [
     numpy
-    python
   ];
 
   installCheckInputs = [
@@ -99,14 +100,6 @@ in rustPlatform.buildRustPackage rec {
       ln -s ${openaiMerges} openai-gpt-merges.txt )
   '';
 
-  buildPhase = ''
-    ${python.interpreter} setup.py bdist_wheel
-  '';
-
-  installPhase = ''
-    pipInstallPhase
-  '';
-
   preCheck = ''
     HOME=$TMPDIR
   '';