{"library":"rustbpe","type":"library","category":null,"description":"RustBPE is a Python library that provides a fast Byte Pair Encoding (BPE) tokenizer implemented in Rust, with Python bindings. It is designed primarily for training GPT-style BPE tokenizers and offers features like parallel processing, GPT-4 style regex pre-tokenization, and direct export to the tiktoken format for efficient inference. Currently at version 0.1.0, it is an initial release, suggesting active and potentially rapid development.","language":"python","status":"active","version":"0.1.0","tags":["BPE","tokenizer","NLP","Rust","Python bindings","LLM","tiktoken","training"],"install":[{"cmd":"pip install rustbpe","imports":["import rustbpe\ntokenizer = rustbpe.Tokenizer()"]}],"homepage":null,"github":null,"docs":null,"changelog":null,"pypi":"https://pypi.org/project/rustbpe/","npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"compatibility":{"summary":{"python_range":"3.10–3.9","success_rate":50,"avg_install_s":1.8,"avg_import_s":0.02,"wheel_type":"wheel"},"url":"https://checklist.day/v1/registry/rustbpe/compatibility"},"provenance":{"verified_status":"passing","verified_at":"Sun Jun 28","last_verified":"Sun Jun 28","next_check":"Tue Jul 28","install_tag":null}}