{"library":"pytorch-tokenizers","type":"library","category":null,"description":"PyTorch-Tokenizers is a Python package providing efficient C++ implementations for common tokenizers like SentencePiece and TikToken, along with Python bindings. It is primarily designed to serve as a dependency for other PyTorch projects, such as ExecuTorch and torchchat, to facilitate building high-performance LLM runners. The library offers significant efficiency gains for AI workloads, multilingual support, and high decode accuracy. It is actively maintained, with version 1.2.0 aligning its releases with major PyTorch and ExecuTorch updates.","language":"python","status":"active","version":"1.2.0","tags":["pytorch","tokenizer","nlp","executorch","sentencepiece","tiktoken","c++"],"install":[{"cmd":"pip install pytorch-tokenizers","imports":["from pytorch_tokenizers import SentencePieceTokenizer","from pytorch_tokenizers import SentencePieceTokenizer"]}],"homepage":"https://pytorch.org/executorch/","github":"https://github.com/pytorch/executorch","docs":null,"changelog":"https://github.com/pytorch/executorch/releases","pypi":"https://pypi.org/project/pytorch-tokenizers/","npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"compatibility":{"summary":{"python_range":"3.10–3.9","success_rate":40,"avg_install_s":6.4,"avg_import_s":null,"wheel_type":"wheel"},"url":"https://checklist.day/v1/registry/pytorch-tokenizers/compatibility"},"provenance":{"verified_status":"install_fail","verified_at":"Sun Jun 28","last_verified":"Sun Jun 28","next_check":"Thu Jul 09","install_tag":null}}