{"library":"sentencepiece","type":"library","category":null,"description":"SentencePiece is an unsupervised text tokenizer and detokenizer, primarily designed for Neural Network-based text generation systems where the vocabulary size is predetermined. It implements subword units like Byte-Pair Encoding (BPE) and Unigram Language Model, capable of training directly from raw sentences without pre-tokenization. The library is actively maintained with regular updates. The current version is 0.2.1.","language":"python","status":"active","version":"0.2.1","tags":["nlp","tokenization","subword","machine-learning"],"install":[{"cmd":"pip install sentencepiece","imports":["import sentencepiece as spm\nsp = spm.SentencePieceProcessor()","import sentencepiece as spm\nspm.SentencePieceTrainer.train(...)"]}],"homepage":null,"github":"https://github.com/google/sentencepiece","docs":null,"changelog":null,"pypi":"https://pypi.org/project/sentencepiece/","npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"compatibility":{"summary":{"python_range":"3.10–3.9","success_rate":50,"avg_install_s":1.6,"avg_import_s":0.06,"wheel_type":"wheel"},"url":"https://checklist.day/v1/registry/sentencepiece/compatibility"},"provenance":{"verified_status":"passing","verified_at":"Sun Jun 28","last_verified":"Sun Jun 28","next_check":"Tue Jul 28","install_tag":"draft"}}