{"library":"pytokens","title":"A Fast, spec compliant Python 3.14+ tokenizer","description":"Pytokens is an open-source Python library providing a fast, spec-compliant tokenizer for Python 3.14+ that is also capable of running on older Python versions (>=3.8). Currently at version 0.4.1, it appears to be actively maintained with recent releases focusing on packaging and development improvements.","language":"python","status":"active","last_verified":"Tue May 19","install":{"commands":["pip install pytokens"],"cli":null},"imports":["import pytokens"],"auth":{"required":false,"env_vars":[]},"quickstart":{"code":"echo \"print('Hello, World!')\" > example.py\npython -m pytokens example.py","lang":"bash","description":"Pytokens is primarily designed for command-line usage to tokenize Python files. This example demonstrates tokenizing a simple Python file. The output will be a stream of tokens (e.g., NAME, STRING, OP, etc.) representing the input file's content.","tag":"stale","tag_description":"widespread failures or data too old to trust","last_tested":"2026-04-23","results":[{"runtime":"python:3.10-alpine","exit_code":1},{"runtime":"python:3.10-slim","exit_code":1},{"runtime":"python:3.11-alpine","exit_code":1},{"runtime":"python:3.11-slim","exit_code":1},{"runtime":"python:3.12-alpine","exit_code":1},{"runtime":"python:3.12-slim","exit_code":1},{"runtime":"python:3.13-alpine","exit_code":1},{"runtime":"python:3.13-slim","exit_code":1},{"runtime":"python:3.9-alpine","exit_code":1},{"runtime":"python:3.9-slim","exit_code":1}]},"compatibility":{"tag":"verified","tag_description":"installs cleanly on critical runtimes, fast import, recently tested","last_tested":"2026-05-19","installed_version":"0.4.1","pypi_latest":"0.4.1","is_stale":false,"summary":{"python_range":"3.10–3.9","success_rate":100,"avg_install_s":1.6,"avg_import_s":0.06,"wheel_type":"wheel"},"results":[{"runtime":"python:3.10-alpine","python_version":"3.10","os_libc":"alpine (musl)","variant":"pytokens","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":0.03,"mem_mb":1.8,"disk_size":"18.5M"},{"runtime":"python:3.10-alpine","python_version":"3.10","os_libc":"alpine (musl)","variant":"pytokens","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.04,"mem_mb":1.8,"disk_size":"18.5M"},{"runtime":"python:3.10-slim","python_version":"3.10","os_libc":"slim (glibc)","variant":"pytokens","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":1.5,"import_time_s":0.02,"mem_mb":1.8,"disk_size":"19M"},{"runtime":"python:3.10-slim","python_version":"3.10","os_libc":"slim (glibc)","variant":"pytokens","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.03,"mem_mb":1.8,"disk_size":"19M"},{"runtime":"python:3.11-alpine","python_version":"3.11","os_libc":"alpine (musl)","variant":"pytokens","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":0.07,"mem_mb":2,"disk_size":"20.4M"},{"runtime":"python:3.11-alpine","python_version":"3.11","os_libc":"alpine (musl)","variant":"pytokens","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.1,"mem_mb":2,"disk_size":"20.4M"},{"runtime":"python:3.11-slim","python_version":"3.11","os_libc":"slim (glibc)","variant":"pytokens","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":1.6,"import_time_s":0.05,"mem_mb":2,"disk_size":"21M"},{"runtime":"python:3.11-slim","python_version":"3.11","os_libc":"slim (glibc)","variant":"pytokens","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.09,"mem_mb":2,"disk_size":"21M"},{"runtime":"python:3.12-alpine","python_version":"3.12","os_libc":"alpine (musl)","variant":"pytokens","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":0.06,"mem_mb":1.8,"disk_size":"12.3M"},{"runtime":"python:3.12-alpine","python_version":"3.12","os_libc":"alpine (musl)","variant":"pytokens","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.1,"mem_mb":1.8,"disk_size":"12.3M"},{"runtime":"python:3.12-slim","python_version":"3.12","os_libc":"slim (glibc)","variant":"pytokens","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":1.4,"import_time_s":0.06,"mem_mb":1.8,"disk_size":"13M"},{"runtime":"python:3.12-slim","python_version":"3.12","os_libc":"slim (glibc)","variant":"pytokens","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.09,"mem_mb":1.8,"disk_size":"13M"},{"runtime":"python:3.13-alpine","python_version":"3.13","os_libc":"alpine (musl)","variant":"pytokens","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":0.05,"mem_mb":2.3,"disk_size":"12.0M"},{"runtime":"python:3.13-alpine","python_version":"3.13","os_libc":"alpine (musl)","variant":"pytokens","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.05,"mem_mb":2.3,"disk_size":"11.9M"},{"runtime":"python:3.13-slim","python_version":"3.13","os_libc":"slim (glibc)","variant":"pytokens","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":1.5,"import_time_s":0.05,"mem_mb":2.1,"disk_size":"13M"},{"runtime":"python:3.13-slim","python_version":"3.13","os_libc":"slim (glibc)","variant":"pytokens","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.08,"mem_mb":2.1,"disk_size":"12M"},{"runtime":"python:3.9-alpine","python_version":"3.9","os_libc":"alpine (musl)","variant":"pytokens","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":0.03,"mem_mb":1.7,"disk_size":"18.0M"},{"runtime":"python:3.9-alpine","python_version":"3.9","os_libc":"alpine (musl)","variant":"pytokens","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.04,"mem_mb":1.7,"disk_size":"18.0M"},{"runtime":"python:3.9-slim","python_version":"3.9","os_libc":"slim (glibc)","variant":"pytokens","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":1.8,"import_time_s":0.02,"mem_mb":1.7,"disk_size":"19M"},{"runtime":"python:3.9-slim","python_version":"3.9","os_libc":"slim (glibc)","variant":"pytokens","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.04,"mem_mb":1.7,"disk_size":"19M"}]}}