public/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #109532 from danieldk/tokenizers-0.10.0

Browse Source 
python3Packages.tokenizers: 0.9.4 -> 0.10.0
This commit is contained in:
Guillaume Girol 2021-01-23 16:56:24 +00:00 committed by GitHub
commit 4f33765575
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pkgs/development/python-modules/tokenizers/default.nix
View File

@ -7,6 +7,7 @@
, wheel , wheel
, numpy , numpy
, python , python
, datasets
, pytestCheckHook , pytestCheckHook
, requests , requests
}: }:
@ -50,16 +51,16 @@ let
}; };
in rustPlatform.buildRustPackage rec { in rustPlatform.buildRustPackage rec {
pname = "tokenizers"; pname = "tokenizers";
version = "0.9.4"; version = "0.10.0";
src = fetchFromGitHub { src = fetchFromGitHub {
owner = "huggingface"; owner = "huggingface";
repo = pname; repo = pname;
rev = "python-v${version}"; rev = "python-v${version}";
hash = "sha256-JXoH9yfhMIFg5qDY5zrF6iWb7XKugjMfk1NxSizfaWg="; hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw=";
}; };
cargoSha256 = "sha256-u9qitrOxJSABs0VjwHUZgmw7VTQXNbp6l8fKKE/RQ7M="; cargoSha256 = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
sourceRoot = "source/bindings/python"; sourceRoot = "source/bindings/python";
@ -75,6 +76,7 @@ in rustPlatform.buildRustPackage rec {
]; ];
installCheckInputs = [ installCheckInputs = [
datasets
pytestCheckHook pytestCheckHook
requests requests
]; ];
@ -105,6 +107,15 @@ in rustPlatform.buildRustPackage rec {
pipInstallPhase pipInstallPhase
''; '';
preCheck = ''
HOME=$TMPDIR
'';
disabledTests = [
# Downloads data using the datasets module.
"TestTrainFromIterators"
];
meta = with lib; { meta = with lib; {
homepage = "https://github.com/huggingface/tokenizers"; homepage = "https://github.com/huggingface/tokenizers";
description = "Fast State-of-the-Art Tokenizers optimized for Research and Production"; description = "Fast State-of-the-Art Tokenizers optimized for Research and Production";