diff --git a/pkgs/development/python-modules/transformers/default.nix b/pkgs/development/python-modules/transformers/default.nix index 743cbebea75..e2409c155c9 100644 --- a/pkgs/development/python-modules/transformers/default.nix +++ b/pkgs/development/python-modules/transformers/default.nix @@ -1,32 +1,28 @@ { buildPythonPackage , lib, stdenv , fetchFromGitHub -, isPy39 +, pythonOlder , cookiecutter , filelock +, importlib-metadata , regex , requests , numpy -, pandas -, parameterized , protobuf , sacremoses -, timeout-decorator , tokenizers , tqdm -, pytestCheckHook }: buildPythonPackage rec { pname = "transformers"; - version = "4.1.1"; - disabled = isPy39; + version = "4.2.1"; src = fetchFromGitHub { owner = "huggingface"; repo = pname; rev = "v${version}"; - sha256 = "1l1gxdsakjmzsgggypq45pnwm87brhlccjfzafs43460pz0wbd6k"; + sha256 = "0yf5s878i6v298wxm4cwkb33qyxz5bdr75jmsnldpdw4ml31c3nn"; }; propagatedBuildInputs = [ @@ -39,63 +35,17 @@ buildPythonPackage rec { sacremoses tokenizers tqdm - ]; + ] ++ stdenv.lib.optionals (pythonOlder "3.8") [ importlib-metadata ]; - checkInputs = [ - pandas - parameterized - pytestCheckHook - timeout-decorator - ]; + # Many tests require internet access. + doCheck = false; postPatch = '' substituteInPlace setup.py \ --replace "tokenizers == 0.9.4" "tokenizers" ''; - preCheck = '' - export HOME="$TMPDIR" - - # This test requires the `datasets` module to download test - # data. However, since we cannot download in the Nix sandbox - # and `dataset` is an optional dependency for transformers - # itself, we will just remove the tests files that import - # `dataset`. - rm tests/test_retrieval_rag.py - rm tests/test_trainer.py - ''; - - # We have to run from the main directory for the tests. However, - # letting pytest discover tests leads to errors. - pytestFlagsArray = [ "tests" ]; - - # Disable tests that require network access. - disabledTests = [ - "BlenderbotSmallTokenizerTest" - "Blenderbot3BTokenizerTests" - "GetFromCacheTests" - "TokenizationTest" - "TestTokenizationBart" - "test_all_tokenizers" - "test_batch_encoding_is_fast" - "test_batch_encoding_pickle" - "test_batch_encoding_word_to_tokens" - "test_config_from_model_shortcut" - "test_config_model_type_from_model_identifier" - "test_from_pretrained_use_fast_toggle" - "test_hf_api" - "test_outputs_can_be_shorter" - "test_outputs_not_longer_than_maxlen" - "test_padding_accepts_tensors" - "test_pretokenized_tokenizers" - "test_tokenizer_equivalence_en_de" - "test_tokenizer_from_model_type" - "test_tokenizer_from_model_type" - "test_tokenizer_from_pretrained" - "test_tokenizer_from_tokenizer_class" - "test_tokenizer_identifier_with_correct_config" - "test_tokenizer_identifier_non_existent" - ]; + pythonImportsCheck = [ "transformers" ]; meta = with lib; { homepage = "https://github.com/huggingface/transformers";