{"library":"robots","type":"library","category":null,"description":"Parses robots.txt files to determine allowed and disallowed paths for web crawlers.","language":"ruby","status":"active","version":"0.10.1","tags":["robots","parsing","web","crawler"],"last_verified":"Fri Jun 19","install":[{"cmd":"gem install robots","imports":["require 'robots'"]},{"cmd":"# Gemfile\ngem 'robots', '~> 0.10.1'","imports":[]}],"homepage":"http://github.com/fizx/robots","github":"https://github.com/fizx/robots","docs":"http://www.rubydoc.info/gems/robots/","changelog":null,"pypi":null,"npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"compatibility":null}