{"library":"tokenizers","type":"library","category":null,"description":"Tokenizers is a Python library providing fast and versatile tokenization tools, optimized for both research and production environments. The current version is 0.22.2, released on January 5, 2026. The library is actively maintained with regular updates to enhance performance and add features.","language":"python","status":"active","version":"0.22.2","tags":["tokenization","NLP","Hugging Face","Python"],"last_verified":"Tue Jun 09","install":[{"cmd":"pip install tokenizers","imports":["from tokenizers import Tokenizer"]}],"homepage":null,"github":"https://github.com/huggingface/tokenizers","docs":null,"changelog":null,"pypi":"https://pypi.org/project/tokenizers/","npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"compatibility":{"summary":{"python_range":"3.10–3.9","success_rate":100,"avg_install_s":5.1,"avg_import_s":0.03,"wheel_type":"wheel"},"url":"https://checklist.day/v1/registry/tokenizers/compatibility"}}