{"library":"tiktoken","type":"library","category":null,"description":"Fast BPE tokenizer from OpenAI, written in Rust. Used to count tokens and encode/decode text for OpenAI models. 3-6x faster than comparable Python tokenizers. Does NOT call any API — purely local computation. Requires a Rust compiler at build time on platforms without pre-built wheels. Package name and import name are both 'tiktoken'.","language":"python","status":"active","version":"0.12.0","tags":["tiktoken","openai","tokenizer","bpe","token-counting","gpt-4o","cl100k","o200k","context-window"],"last_verified":"Tue Jun 09","install":[{"cmd":"pip install tiktoken","imports":["import tiktoken; enc = tiktoken.get_encoding('o200k_base')","import tiktoken; enc = tiktoken.encoding_for_model('gpt-4o')"]},{"cmd":"pip install tiktoken --no-binary tiktoken","imports":[]}],"homepage":null,"github":"https://github.com/openai/tiktoken","docs":null,"changelog":"https://github.com/openai/tiktoken/blob/main/CHANGELOG.md","pypi":"https://pypi.org/project/tiktoken/","npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"compatibility":{"summary":{"python_range":"3.10–3.9","success_rate":50,"avg_install_s":2.9,"avg_import_s":4.23,"wheel_type":"wheel"},"url":"https://checklist.day/v1/registry/tiktoken/compatibility"}}