{"library":"temoto-robotstxt","type":"library","category":null,"description":"Package robotstxt implements the robots.txt Exclusion Protocol with various extensions.","language":"go","status":"active","version":"1.1.2","tags":["robots.txt","web-scraping","crawler"],"last_verified":"Mon Jun 15","install":[{"cmd":"go get github.com/temoto/robotstxt","imports":["github.com/temoto/robotstxt"]}],"homepage":"https://github.com/temoto/robotstxt","github":"https://github.com/temoto/robotstxt","docs":"https://pkg.go.dev/github.com/temoto/robotstxt#section-documentation","changelog":null,"pypi":null,"npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"compatibility":null}