public/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

python3Packages.tokenizers: 0.9.4 -> 0.10.0

Browse Source 
Changelog:
https://github.com/huggingface/tokenizers/releases/tag/python-v0.10.0
This commit is contained in:
Daniël de Kok 2021-01-16 15:07:33 +01:00
parent bcdca92787
commit cca0b0117b
1 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pkgs/development/python-modules/tokenizers/default.nix
View File

@ -7,6 +7,7 @@
, wheel , wheel
, numpy , numpy
, python , python
, datasets
, pytestCheckHook , pytestCheckHook
, requests , requests
}: }:
@ -50,16 +51,16 @@ let
}; };
in rustPlatform.buildRustPackage rec { in rustPlatform.buildRustPackage rec {
pname = "tokenizers"; pname = "tokenizers";
version = "0.9.4"; version = "0.10.0";
src = fetchFromGitHub { src = fetchFromGitHub {
owner = "huggingface"; owner = "huggingface";
repo = pname; repo = pname;
rev = "python-v${version}"; rev = "python-v${version}";
hash = "sha256-JXoH9yfhMIFg5qDY5zrF6iWb7XKugjMfk1NxSizfaWg="; hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw=";
}; };
cargoSha256 = "sha256-u9qitrOxJSABs0VjwHUZgmw7VTQXNbp6l8fKKE/RQ7M="; cargoSha256 = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
sourceRoot = "source/bindings/python"; sourceRoot = "source/bindings/python";
@ -75,6 +76,7 @@ in rustPlatform.buildRustPackage rec {
]; ];
installCheckInputs = [ installCheckInputs = [
datasets
pytestCheckHook pytestCheckHook
requests requests
]; ];
@ -105,6 +107,15 @@ in rustPlatform.buildRustPackage rec {
pipInstallPhase pipInstallPhase
''; '';
preCheck = ''
HOME=$TMPDIR
'';
disabledTests = [
# Downloads data using the datasets module.
"TestTrainFromIterators"
];
meta = with lib; { meta = with lib; {
homepage = "https://github.com/huggingface/tokenizers"; homepage = "https://github.com/huggingface/tokenizers";
description = "Fast State-of-the-Art Tokenizers optimized for Research and Production"; description = "Fast State-of-the-Art Tokenizers optimized for Research and Production";