{"library":"robotspy","type":"library","category":null,"description":"robotspy is a Python library for parsing `robots.txt` files, implementing the Robots Exclusion Protocol (REP) as defined by RFC 9309. It allows applications to determine whether a web crawler is permitted to access a given URL path on a server. The library is actively maintained, with the current version being 0.13.0, and has a steady release cadence addressing bug fixes and adherence to Google's parsing behavior.","language":"python","status":"active","version":"0.13.0","tags":["web scraping","robots.txt","parser","SEO","web crawling"],"install":[{"cmd":"pip install robotspy","imports":["from robots import RobotFileParser","from robots import RobotsParser","from robots import RequestRate"]}],"homepage":null,"github":"https://github.com/andreburgaud/robotspy","docs":null,"changelog":null,"pypi":"https://pypi.org/project/robotspy/","npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"compatibility":{"summary":{"python_range":"3.10–3.9","success_rate":100,"avg_install_s":1.5,"avg_import_s":null,"wheel_type":"wheel"},"url":"https://checklist.day/v1/registry/robotspy/compatibility"},"provenance":{"verified_status":"passing","verified_at":"Tue Jun 30","last_verified":"Tue Jun 30","next_check":"Thu Jul 30","install_tag":null}}