{"library":"robotstxt","type":"library","category":null,"description":"Robotstxt Parser allows you to check the accessibility of URLs and get other data. Full support for the robots.txt RFC, wildcards and Sitemap rules.","language":"ruby","status":"active","version":"0.5.4","tags":["robots.txt","parser","seo","web"],"last_verified":"Fri Jun 19","install":[{"cmd":"gem install robotstxt","imports":["require 'robotstxt'"]},{"cmd":"# Gemfile\ngem 'robotstxt', '~> 0.5.4'","imports":[]}],"homepage":"http://www.simonerinzivillo.it","github":"https://github.com/rinzi/robotstxt","docs":"http://www.rubydoc.info/gems/robotstxt/","changelog":null,"pypi":null,"npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"compatibility":null}