{"library":"sacremoses","title":"SacreMoses","description":"SacreMoses is a Python port of the widely-used Moses tokenizer, truecaser, and punctuation normalizer tools, essential for many Natural Language Processing (NLP) tasks, particularly in machine translation workflows. The current version is 0.1.1. Releases are made periodically, addressing bug fixes, performance improvements, and alignment with the original Perl implementation.","language":"python","status":"active","last_verified":"Thu May 21","install":{"commands":["pip install sacremoses"],"cli":{"name":"sacremoses","version":"sacremoses, version 0.1.1"}},"imports":["from sacremoses import MosesTokenizer","from sacremoses import MosesDetokenizer","from sacremoses import MosesPunctNormalizer","from sacremoses import MosesTruecaser"],"auth":{"required":false,"env_vars":[]},"quickstart":{"code":"from sacremoses import MosesTokenizer, MosesDetokenizer, MosesPunctNormalizer\n\n# Punctuation Normalization\nmpn = MosesPunctNormalizer(lang='en')\ntext_to_normalize = 'THIS EBOOK IS OTHERWISE PROVIDED TO YOU \"AS-IS.\"'\nnormalized_text = mpn.normalize(text_to_normalize)\nprint(f\"Normalized: {normalized_text}\")\n\n# Tokenization and Detokenization\nmt = MosesTokenizer(lang='en')\nmd = MosesDetokenizer(lang='en')\n\nsample_text = \"Hello, world! This is a test sentence with numbers 123 and some special characters like @#$%.\" \ntokenized_list = mt.tokenize(sample_text)\ndetokenized_text = md.detokenize(tokenized_list)\n\nprint(f\"Original: {sample_text}\")\nprint(f\"Tokenized: {tokenized_list}\")\nprint(f\"Detokenized: {detokenized_text}\")","lang":"python","description":"This quickstart demonstrates the core functionalities: punctuation normalization and the tokenization/detokenization of a sample English sentence.","tag":null,"tag_description":null,"last_tested":"2026-04-25","results":[{"runtime":"python:3.10-alpine","exit_code":0},{"runtime":"python:3.10-slim","exit_code":0},{"runtime":"python:3.11-alpine","exit_code":0},{"runtime":"python:3.11-slim","exit_code":0},{"runtime":"python:3.12-alpine","exit_code":0},{"runtime":"python:3.12-slim","exit_code":0},{"runtime":"python:3.13-alpine","exit_code":0},{"runtime":"python:3.13-slim","exit_code":0},{"runtime":"python:3.9-alpine","exit_code":0},{"runtime":"python:3.9-slim","exit_code":0}]},"compatibility":{"tag":null,"tag_description":null,"last_tested":"2026-05-21","installed_version":"0.1.1","pypi_latest":"0.1.1","is_stale":false,"summary":{"python_range":"3.10–3.9","success_rate":100,"avg_install_s":2.6,"avg_import_s":2,"wheel_type":"wheel"},"results":[{"runtime":"python:3.10-alpine","python_version":"3.10","os_libc":"alpine (musl)","variant":"sacremoses","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":1.26,"mem_mb":15.8,"disk_size":"25.9M"},{"runtime":"python:3.10-alpine","python_version":"3.10","os_libc":"alpine (musl)","variant":"sacremoses","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":1.67,"mem_mb":15.8,"disk_size":"25.9M"},{"runtime":"python:3.10-slim","python_version":"3.10","os_libc":"slim (glibc)","variant":"sacremoses","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":2.7,"import_time_s":0.79,"mem_mb":15.8,"disk_size":"27M"},{"runtime":"python:3.10-slim","python_version":"3.10","os_libc":"slim (glibc)","variant":"sacremoses","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.97,"mem_mb":15.8,"disk_size":"27M"},{"runtime":"python:3.11-alpine","python_version":"3.11","os_libc":"alpine (musl)","variant":"sacremoses","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":2.6,"mem_mb":17.4,"disk_size":"28.8M"},{"runtime":"python:3.11-alpine","python_version":"3.11","os_libc":"alpine (musl)","variant":"sacremoses","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":3.42,"mem_mb":17.4,"disk_size":"28.8M"},{"runtime":"python:3.11-slim","python_version":"3.11","os_libc":"slim (glibc)","variant":"sacremoses","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":2.5,"import_time_s":2.25,"mem_mb":17.4,"disk_size":"30M"},{"runtime":"python:3.11-slim","python_version":"3.11","os_libc":"slim (glibc)","variant":"sacremoses","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":2.75,"mem_mb":17.4,"disk_size":"30M"},{"runtime":"python:3.12-alpine","python_version":"3.12","os_libc":"alpine (musl)","variant":"sacremoses","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":2.06,"mem_mb":17.8,"disk_size":"20.4M"},{"runtime":"python:3.12-alpine","python_version":"3.12","os_libc":"alpine (musl)","variant":"sacremoses","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":2.61,"mem_mb":17.8,"disk_size":"20.4M"},{"runtime":"python:3.12-slim","python_version":"3.12","os_libc":"slim (glibc)","variant":"sacremoses","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":2.2,"import_time_s":2.19,"mem_mb":17.8,"disk_size":"21M"},{"runtime":"python:3.12-slim","python_version":"3.12","os_libc":"slim (glibc)","variant":"sacremoses","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":2.77,"mem_mb":17.8,"disk_size":"21M"},{"runtime":"python:3.13-alpine","python_version":"3.13","os_libc":"alpine (musl)","variant":"sacremoses","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":2.06,"mem_mb":18,"disk_size":"20.2M"},{"runtime":"python:3.13-alpine","python_version":"3.13","os_libc":"alpine (musl)","variant":"sacremoses","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":2.46,"mem_mb":18,"disk_size":"20.1M"},{"runtime":"python:3.13-slim","python_version":"3.13","os_libc":"slim (glibc)","variant":"sacremoses","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":2.2,"import_time_s":2.01,"mem_mb":18,"disk_size":"21M"},{"runtime":"python:3.13-slim","python_version":"3.13","os_libc":"slim (glibc)","variant":"sacremoses","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":3.08,"mem_mb":18,"disk_size":"21M"},{"runtime":"python:3.9-alpine","python_version":"3.9","os_libc":"alpine (musl)","variant":"sacremoses","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":1.16,"mem_mb":15.5,"disk_size":"25.3M"},{"runtime":"python:3.9-alpine","python_version":"3.9","os_libc":"alpine (musl)","variant":"sacremoses","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":1.56,"mem_mb":15.5,"disk_size":"25.3M"},{"runtime":"python:3.9-slim","python_version":"3.9","os_libc":"slim (glibc)","variant":"sacremoses","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":3.1,"import_time_s":0.91,"mem_mb":15.5,"disk_size":"26M"},{"runtime":"python:3.9-slim","python_version":"3.9","os_libc":"slim (glibc)","variant":"sacremoses","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":1.45,"mem_mb":15.5,"disk_size":"26M"}]}}