{"library":"scrapy","title":"Scrapy","description":"Scrapy is a high-level Python web crawling and web scraping framework, designed for fast extraction of structured data from websites. It's actively maintained with frequent releases, supporting applications from data mining to information processing and automated testing. The current version is 2.15.0.","language":"python","status":"active","last_verified":"Thu May 21","install":{"commands":["pip install scrapy"],"cli":{"name":"scrapy","version":"Scrapy 2.15.2 - no active project"}},"imports":["import scrapy","from scrapy import Spider","from scrapy import Request","from scrapy import Item","from scrapy.item import Field","from scrapy.spiders import CrawlSpider","from scrapy.linkextractors import LinkExtractor","from scrapy.crawler import AsyncCrawlerProcess"],"auth":{"required":false,"env_vars":[]},"quickstart":{"code":"import scrapy\n\nclass QuotesSpider(scrapy.Spider):\n    name = \"quotes\"\n    start_urls = [\n        \"https://quotes.toscrape.com/tag/humor/\",\n    ]\n\n    def parse(self, response):\n        for quote in response.css(\"div.quote\"):\n            yield {\n                \"author\": quote.xpath(\"span/small/text()\").get(),\n                \"text\": quote.css(\"span.text::text\").get(),\n            }\n\n        next_page = response.css('li.next a::attr(\"href\")').get()\n        if next_page is not None:\n            yield response.follow(next_page, self.parse)\n\n# To run this spider, save it as a .py file (e.g., quotes_spider.py) and execute:\n# scrapy runspider quotes_spider.py -o quotes.jsonl","lang":"python","description":"This quickstart demonstrates a basic Scrapy spider that crawls the 'quotes.toscrape.com' website, specifically the 'humor' tag. It extracts the author and text of each quote, then follows the 'Next Page' link to continue crawling. The `start_urls` attribute defines the initial URLs, and the `parse` method handles the response, extracting data and scheduling new requests using `response.follow` for pagination.","tag":null,"tag_description":null,"last_tested":"2026-04-25","results":[{"runtime":"python:3.10-alpine","exit_code":0},{"runtime":"python:3.10-slim","exit_code":0},{"runtime":"python:3.11-alpine","exit_code":0},{"runtime":"python:3.11-slim","exit_code":0},{"runtime":"python:3.12-alpine","exit_code":0},{"runtime":"python:3.12-slim","exit_code":0},{"runtime":"python:3.13-alpine","exit_code":0},{"runtime":"python:3.13-slim","exit_code":0},{"runtime":"python:3.9-alpine","exit_code":0},{"runtime":"python:3.9-slim","exit_code":0}]},"compatibility":{"tag":null,"tag_description":null,"last_tested":"2026-05-21","installed_version":"2.13.4","pypi_latest":"2.16.0","is_stale":true,"summary":{"python_range":"3.10–3.9","success_rate":100,"avg_install_s":7.3,"avg_import_s":1.47,"wheel_type":"wheel"},"results":[{"runtime":"python:3.10-alpine","python_version":"3.10","os_libc":"alpine (musl)","variant":"scrapy","exit_code":0,"wheel_type":"sdist","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":1.25,"mem_mb":22.1,"disk_size":"91.5M"},{"runtime":"python:3.10-alpine","python_version":"3.10","os_libc":"alpine (musl)","variant":"scrapy","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":1.42,"mem_mb":22.1,"disk_size":"91.3M"},{"runtime":"python:3.10-slim","python_version":"3.10","os_libc":"slim (glibc)","variant":"scrapy","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":7.6,"import_time_s":0.93,"mem_mb":22.1,"disk_size":"92M"},{"runtime":"python:3.10-slim","python_version":"3.10","os_libc":"slim (glibc)","variant":"scrapy","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.97,"mem_mb":22.1,"disk_size":"92M"},{"runtime":"python:3.11-alpine","python_version":"3.11","os_libc":"alpine (musl)","variant":"scrapy","exit_code":0,"wheel_type":"sdist","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":1.66,"mem_mb":24.4,"disk_size":"103.4M"},{"runtime":"python:3.11-alpine","python_version":"3.11","os_libc":"alpine (musl)","variant":"scrapy","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":1.77,"mem_mb":24.4,"disk_size":"103.2M"},{"runtime":"python:3.11-slim","python_version":"3.11","os_libc":"slim (glibc)","variant":"scrapy","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":7.4,"import_time_s":1.44,"mem_mb":24.4,"disk_size":"104M"},{"runtime":"python:3.11-slim","python_version":"3.11","os_libc":"slim (glibc)","variant":"scrapy","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":1.45,"mem_mb":24.4,"disk_size":"104M"},{"runtime":"python:3.12-alpine","python_version":"3.12","os_libc":"alpine (musl)","variant":"scrapy","exit_code":0,"wheel_type":"sdist","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":1.68,"mem_mb":24,"disk_size":"93.1M"},{"runtime":"python:3.12-alpine","python_version":"3.12","os_libc":"alpine (musl)","variant":"scrapy","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":1.78,"mem_mb":24,"disk_size":"92.8M"},{"runtime":"python:3.12-slim","python_version":"3.12","os_libc":"slim (glibc)","variant":"scrapy","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":6.5,"import_time_s":1.7,"mem_mb":24,"disk_size":"94M"},{"runtime":"python:3.12-slim","python_version":"3.12","os_libc":"slim (glibc)","variant":"scrapy","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":1.89,"mem_mb":24,"disk_size":"93M"},{"runtime":"python:3.13-alpine","python_version":"3.13","os_libc":"alpine (musl)","variant":"scrapy","exit_code":0,"wheel_type":"sdist","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":1.73,"mem_mb":24.6,"disk_size":"92.5M"},{"runtime":"python:3.13-alpine","python_version":"3.13","os_libc":"alpine (musl)","variant":"scrapy","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":1.71,"mem_mb":24.6,"disk_size":"92.2M"},{"runtime":"python:3.13-slim","python_version":"3.13","os_libc":"slim (glibc)","variant":"scrapy","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":6.5,"import_time_s":1.67,"mem_mb":24.6,"disk_size":"93M"},{"runtime":"python:3.13-slim","python_version":"3.13","os_libc":"slim (glibc)","variant":"scrapy","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":1.89,"mem_mb":24.6,"disk_size":"93M"},{"runtime":"python:3.9-alpine","python_version":"3.9","os_libc":"alpine (musl)","variant":"scrapy","exit_code":0,"wheel_type":"sdist","failure_reason":null,"import_side_effects":"clean","install_time_s":null,"import_time_s":1.1,"mem_mb":22.1,"disk_size":"91.3M"},{"runtime":"python:3.9-alpine","python_version":"3.9","os_libc":"alpine (musl)","variant":"scrapy","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":1.28,"mem_mb":22,"disk_size":"91.1M"},{"runtime":"python:3.9-slim","python_version":"3.9","os_libc":"slim (glibc)","variant":"scrapy","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":8.7,"import_time_s":1.06,"mem_mb":22.1,"disk_size":"92M"},{"runtime":"python:3.9-slim","python_version":"3.9","os_libc":"slim (glibc)","variant":"scrapy","exit_code":0,"wheel_type":null,"failure_reason":null,"import_side_effects":null,"install_time_s":null,"import_time_s":0.99,"mem_mb":22,"disk_size":"92M"}]}}