{"library":"protego","title":"Protego","description":"Protego is a pure-Python robots.txt parser with support for modern conventions like those defined by Google. As of version 0.6.0, it actively supports Python 3.10 and newer, with regular updates aligning with new Python releases. It is widely used for web scraping and compliance checking.","language":"python","status":"active","last_verified":"Wed May 20","install":{"commands":["pip install protego"],"cli":null},"imports":["from protego import Protego"],"auth":{"required":false,"env_vars":[]},"quickstart":{"code":"from protego import Protego\n\nrobotstxt_content = \"\"\"\nUser-agent: *\nDisallow: /admin/\nAllow: /admin/login\nCrawl-delay: 5\nSitemap: http://example.com/sitemap.xml\n\"\"\"\n\nrp = Protego.parse(robotstxt_content)\n\n# Check if a URL can be fetched by a user agent\ncan_fetch_admin = rp.can_fetch(\"http://example.com/admin/settings\", \"mybot\")\ncan_fetch_login = rp.can_fetch(\"http://example.com/admin/login\", \"mybot\")\n\nprint(f\"Can 'mybot' fetch /admin/settings? {can_fetch_admin}\")\nprint(f\"Can 'mybot' fetch /admin/login? {can_fetch_login}\")\nprint(f\"Crawl delay for 'mybot': {rp.crawl_delay('mybot')} seconds\")\nprint(f\"Sitemaps: {list(rp.sitemaps)}\")","lang":"python","description":"Initialize the parser with robots.txt content and check URL access permissions for a specific user agent, retrieve crawl delay, and sitemaps.","tag":null,"tag_description":null,"last_tested":"2026-04-24","results":[{"runtime":"python:3.10-alpine","exit_code":0},{"runtime":"python:3.10-slim","exit_code":0},{"runtime":"python:3.11-alpine","exit_code":0},{"runtime":"python:3.11-slim","exit_code":0},{"runtime":"python:3.12-alpine","exit_code":0},{"runtime":"python:3.12-slim","exit_code":0},{"runtime":"python:3.13-alpine","exit_code":0},{"runtime":"python:3.13-slim","exit_code":0},{"runtime":"python:3.9-alpine","exit_code":0},{"runtime":"python:3.9-slim","exit_code":0}]},"compatibility":{"tag":"verified","tag_description":"installs cleanly on critical runtimes, fast import, recently tested","last_tested":"2026-05-20","installed_version":"0.5.0","pypi_latest":"0.6.0","is_stale":true,"summary":{"python_range":"3.10–3.9","success_rate":100,"avg_install_s":1.5,"avg_import_s":0.05,"wheel_type":"wheel"},"results":[{"runtime":"python:3.10-alpine","python_version":"3.10","os_libc":"alpine (musl)","variant":"protego","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":0.03,"mem_mb":1.8,"disk_size":"17.8M"},{"runtime":"python:3.10-alpine","python_version":"3.10","os_libc":"alpine (musl)","variant":"protego","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.03,"mem_mb":1.8,"disk_size":"17.8M"},{"runtime":"python:3.10-slim","python_version":"3.10","os_libc":"slim (glibc)","variant":"protego","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":1.4,"import_time_s":0.02,"mem_mb":1.8,"disk_size":"18M"},{"runtime":"python:3.10-slim","python_version":"3.10","os_libc":"slim (glibc)","variant":"protego","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.02,"mem_mb":1.8,"disk_size":"18M"},{"runtime":"python:3.11-alpine","python_version":"3.11","os_libc":"alpine (musl)","variant":"protego","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":0.07,"mem_mb":2.2,"disk_size":"19.7M"},{"runtime":"python:3.11-alpine","python_version":"3.11","os_libc":"alpine (musl)","variant":"protego","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.08,"mem_mb":2.2,"disk_size":"19.7M"},{"runtime":"python:3.11-slim","python_version":"3.11","os_libc":"slim (glibc)","variant":"protego","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":1.6,"import_time_s":0.06,"mem_mb":2.2,"disk_size":"20M"},{"runtime":"python:3.11-slim","python_version":"3.11","os_libc":"slim (glibc)","variant":"protego","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.06,"mem_mb":2.2,"disk_size":"20M"},{"runtime":"python:3.12-alpine","python_version":"3.12","os_libc":"alpine (musl)","variant":"protego","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":0.05,"mem_mb":1.8,"disk_size":"11.6M"},{"runtime":"python:3.12-alpine","python_version":"3.12","os_libc":"alpine (musl)","variant":"protego","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.05,"mem_mb":1.8,"disk_size":"11.6M"},{"runtime":"python:3.12-slim","python_version":"3.12","os_libc":"slim (glibc)","variant":"protego","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":1.4,"import_time_s":0.05,"mem_mb":1.8,"disk_size":"12M"},{"runtime":"python:3.12-slim","python_version":"3.12","os_libc":"slim (glibc)","variant":"protego","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.07,"mem_mb":1.8,"disk_size":"12M"},{"runtime":"python:3.13-alpine","python_version":"3.13","os_libc":"alpine (musl)","variant":"protego","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":0.06,"mem_mb":2.1,"disk_size":"11.3M"},{"runtime":"python:3.13-alpine","python_version":"3.13","os_libc":"alpine (musl)","variant":"protego","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.05,"mem_mb":2.1,"disk_size":"11.2M"},{"runtime":"python:3.13-slim","python_version":"3.13","os_libc":"slim (glibc)","variant":"protego","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":1.4,"import_time_s":0.05,"mem_mb":1.9,"disk_size":"12M"},{"runtime":"python:3.13-slim","python_version":"3.13","os_libc":"slim (glibc)","variant":"protego","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.05,"mem_mb":1.9,"disk_size":"12M"},{"runtime":"python:3.9-alpine","python_version":"3.9","os_libc":"alpine (musl)","variant":"protego","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":0.03,"mem_mb":1.8,"disk_size":"17.3M"},{"runtime":"python:3.9-alpine","python_version":"3.9","os_libc":"alpine (musl)","variant":"protego","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.04,"mem_mb":1.8,"disk_size":"17.3M"},{"runtime":"python:3.9-slim","python_version":"3.9","os_libc":"slim (glibc)","variant":"protego","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":1.6,"import_time_s":0.03,"mem_mb":1.8,"disk_size":"18M"},{"runtime":"python:3.9-slim","python_version":"3.9","os_libc":"slim (glibc)","variant":"protego","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.03,"mem_mb":1.8,"disk_size":"18M"}]}}