{"library":"openvino-tokenizers","type":"library","category":null,"description":"OpenVINO Tokenizers provides utilities to convert pre-trained tokenizers, primarily from the Hugging Face `transformers` library, into OpenVINO models. These converted tokenizers can then be compiled and run efficiently on various hardware using the OpenVINO runtime, preparing text inputs for OpenVINO-optimized Large Language Models (LLMs). The current version is 2026.1.0.0, and releases typically align with major OpenVINO toolkit releases, often on a quarterly or yearly cadence.","language":"en","status":"active","version":"2026.1.0.0","tags":["OpenVINO","NLP","Tokenizers","AI","ML","Hugging Face","Transformers","Inference"],"last_verified":"Wed May 27","install":[{"cmd":"pip install openvino-tokenizers","imports":["from openvino_tokenizers import convert_tokenizer"]},{"cmd":"pip install transformers","imports":[]}],"homepage":null,"github":null,"docs":null,"changelog":null,"pypi":null,"npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"compatibility":{"summary":{"python_range":"3.10–3.9","success_rate":75,"avg_install_s":9.8,"avg_import_s":0.74,"wheel_type":"wheel"},"url":"https://checklist.day/v1/registry/openvino-tokenizers/compatibility"}}