{"library":"robotex","type":"library","category":null,"description":"A Ruby library for parsing and obeying robots.txt files to control web crawler access.","language":"ruby","status":"active","version":"1.0.0","tags":["robots","web-crawling","scraping","utility"],"last_verified":"Fri Jun 19","install":[{"cmd":"gem install robotex","imports":["require 'robotex'"]},{"cmd":"# Gemfile\ngem 'robotex', '~> 1.0.0'","imports":[]}],"homepage":"http://www.github.com/chriskite/robotex","github":"https://github.com/chriskite/robotex","docs":"http://www.rubydoc.info/gems/robotex/","changelog":null,"pypi":null,"npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"compatibility":null}