{"id":18354,"library":"express-robots-middleware","title":"express-robots-middleware","description":"Middleware for Express.js applications to dynamically generate a robots.txt file. Version 1.0.1 is the current stable release, with infrequent updates since 2016. It supports multiple user agents, allow/disallow directives, and crawl-delay configuration. Differentiators include simple integration as route middleware and no external dependencies beyond Express.","status":"active","version":"1.0.1","language":"javascript","source_language":"en","source_url":"https://github.com/chriswalkr/express-robots-middleware","tags":["javascript","express","robots.txt","middleware"],"install":[{"cmd":"npm install express-robots-middleware","lang":"bash","label":"npm"},{"cmd":"yarn add express-robots-middleware","lang":"bash","label":"yarn"},{"cmd":"pnpm add express-robots-middleware","lang":"bash","label":"pnpm"}],"dependencies":[{"reason":"peer dependency for providing request/response objects","package":"express","optional":false}],"imports":[{"note":"Default export; no named export.","wrong":"const expressRobotsMiddleware = require('express-robots-middleware').default","symbol":"expressRobotsMiddleware","correct":"import expressRobotsMiddleware from 'express-robots-middleware'"}],"quickstart":{"code":"import express from 'express';\nimport expressRobotsMiddleware from 'express-robots-middleware';\n\nconst app = express();\n\nconst robotsMiddleware = expressRobotsMiddleware({\n  UserAgent: '*',\n  Disallow: ['/private', '/tmp'],\n  Allow: '/',\n  CrawlDelay: '10'\n});\n\napp.get('/robots.txt', robotsMiddleware);\n\napp.listen(3000, () => console.log('Server running on port 3000'));","lang":"typescript","description":"Creates an Express app that serves a robots.txt with a disallow list and crawl delay."},"warnings":[{"fix":"Use exactly { UserAgent: string, Disallow: string[], Allow: string, CrawlDelay: string }","message":"Options must be passed as an object with capitalized keys: UserAgent, Disallow, Allow, CrawlDelay. Using lowercase keys will be silently ignored.","severity":"gotcha","affected_versions":"*"},{"fix":"Upgrade to 1.0.1 to avoid unexpected behavior with Express error middleware.","message":"In version 1.0.0, the middleware returned a function that directly sent the response; in 1.0.1, it uses next() for compatibility with error handling.","severity":"breaking","affected_versions":"1.0.0"},{"fix":"Always provide all required fields: UserAgent, Disallow array, Allow, CrawlDelay.","message":"The middleware does not validate the options. Invalid or missing fields will result in an empty robots.txt or runtime errors.","severity":"gotcha","affected_versions":"*"}],"env_vars":null,"last_verified":"2026-04-25T00:00:00.000Z","next_check":"2026-07-24T00:00:00.000Z","problems":[{"fix":"Bind the middleware to the '/robots.txt' route using app.get('/robots.txt', robotsMiddleware).","cause":"Middleware used without being bound to a route (e.g., app.use(robotsMiddleware) instead of app.get('/robots.txt', robotsMiddleware)).","error":"TypeError: Cannot read property 'set' of undefined"},{"fix":"Use const expressRobotsMiddleware = require('express-robots-middleware').default;","cause":"Default import used incorrectly in CommonJS environment without the .default property.","error":"TypeError: expressRobotsMiddleware is not a function"}],"ecosystem":"npm","meta_description":null,"install_score":null,"install_tag":null,"quickstart_score":null,"quickstart_tag":null}