{"library":"scrapy-playwright","title":"Scrapy Playwright","description":"Scrapy Playwright is a Scrapy Download Handler that integrates Playwright for Python, enabling Scrapy spiders to effectively scrape dynamic web pages that rely heavily on JavaScript rendering. It allows for browser automation within the Scrapy framework, facilitating interaction with complex web elements, while maintaining Scrapy's efficient crawling and scheduling model. The library is actively maintained with frequent releases, currently at version 0.0.46.","language":"python","status":"active","last_verified":"Sat May 16","install":{"commands":["pip install scrapy-playwright playwright"],"cli":{"name":"scrapy","version":"Scrapy 2.15.2 - no active project"}},"imports":["from scrapy_playwright.handler import ScrapyPlaywrightDownloadHandler","from scrapy_playwright.page import PageMethod","from scrapy_playwright.request import PlaywrightRequest"],"auth":{"required":false,"env_vars":[]},"quickstart":{"code":"import scrapy\nfrom scrapy_playwright.page import PageMethod\n\n# settings.py configuration (add these to your project's settings.py)\n# DOWNLOAD_HANDLERS = {\n#     \"http\": \"scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler\",\n#     \"https\": \"scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler\",\n# }\n# TWISTED_REACTOR = \"twisted.internet.asyncioreactor.AsyncioSelectorReactor\"\n# PLAYWRIGHT_BROWSER_TYPE = \"chromium\" # or 'firefox', 'webkit'\n# PLAYWRIGHT_LAUNCH_OPTIONS = {\n#     \"headless\": True, # Set to False for visual debugging\n#     \"timeout\": 60000 # 60 seconds\n# }\n\nclass MySpider(scrapy.Spider):\n    name = \"my_spider\"\n    start_urls = [\"https://quotes.toscrape.com/js/\"]\n\n    def start_requests(self):\n        for url in self.start_urls:\n            yield scrapy.Request(\n                url,\n                meta={\n                    \"playwright\": True,\n                    \"playwright_page_methods\": [\n                        PageMethod(\"wait_for_selector\", \"div.quote\"),\n                        # PageMethod(\"screenshot\", path=\"screenshot.png\", full_page=True),\n                    ],\n                },\n                callback=self.parse_quotes,\n            )\n\n    async def parse_quotes(self, response):\n        # The response is now a PlaywrightResponse object with rendered content\n        for quote in response.css('div.quote'):\n            yield {\n                'text': quote.css('span.text::text').get(),\n                'author': quote.css('small.author::text').get(),\n                'tags': quote.css('div.tags a.tag::text').getall(),\n            }\n\n        next_page = response.css('li.next a::attr(href)').get()\n        if next_page is not None:\n            yield scrapy.Request(response.urljoin(next_page), meta={'playwright': True, 'playwright_page_methods': [PageMethod(\"wait_for_selector\", \"div.quote\")]}, callback=self.parse_quotes)\n","lang":"python","description":"To use Scrapy Playwright, you must configure your `settings.py` file to include the `ScrapyPlaywrightDownloadHandler` and set the `TWISTED_REACTOR` for asyncio compatibility. Then, within your spider, create `scrapy.Request` objects with `meta={'playwright': True}`. You can also define page interactions using `playwright_page_methods` with `PageMethod` objects to wait for elements or perform actions before parsing.","tag":null,"tag_description":null,"last_tested":null,"results":[]},"compatibility":{"tag":null,"tag_description":null,"last_tested":"2026-05-16","installed_version":"0.0.44","pypi_latest":"0.0.46","is_stale":true,"summary":{"python_range":"3.10–3.9","success_rate":50,"avg_install_s":9.6,"avg_import_s":1.77,"wheel_type":"wheel"},"results":[{"runtime":"python:3.10-alpine","python_version":"3.10","os_libc":"alpine (musl)","variant":"scrapy-playwright","exit_code":1,"wheel_type":null,"failure_reason":"build_error","import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.10-slim","python_version":"3.10","os_libc":"slim (glibc)","variant":"scrapy-playwright","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":9.7,"import_time_s":1.3,"mem_mb":28.9,"disk_size":"230M"},{"runtime":"python:3.11-alpine","python_version":"3.11","os_libc":"alpine (musl)","variant":"scrapy-playwright","exit_code":1,"wheel_type":null,"failure_reason":"build_error","import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.11-slim","python_version":"3.11","os_libc":"slim (glibc)","variant":"scrapy-playwright","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":9.3,"import_time_s":1.95,"mem_mb":31.2,"disk_size":"243M"},{"runtime":"python:3.12-alpine","python_version":"3.12","os_libc":"alpine (musl)","variant":"scrapy-playwright","exit_code":1,"wheel_type":null,"failure_reason":"build_error","import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.12-slim","python_version":"3.12","os_libc":"slim (glibc)","variant":"scrapy-playwright","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":8.6,"import_time_s":2.2,"mem_mb":30.8,"disk_size":"232M"},{"runtime":"python:3.13-alpine","python_version":"3.13","os_libc":"alpine (musl)","variant":"scrapy-playwright","exit_code":1,"wheel_type":null,"failure_reason":"build_error","import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.13-slim","python_version":"3.13","os_libc":"slim (glibc)","variant":"scrapy-playwright","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":9,"import_time_s":2.08,"mem_mb":32.5,"disk_size":"232M"},{"runtime":"python:3.9-alpine","python_version":"3.9","os_libc":"alpine (musl)","variant":"scrapy-playwright","exit_code":1,"wheel_type":null,"failure_reason":"build_error","import_side_effects":null,"install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":null},{"runtime":"python:3.9-slim","python_version":"3.9","os_libc":"slim (glibc)","variant":"scrapy-playwright","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"clean","install_time_s":11.2,"import_time_s":1.3,"mem_mb":28.3,"disk_size":"230M"}]}}