{"library":"pytest-spark","title":"pytest-spark","description":"pytest-spark is a pytest plugin that simplifies testing PySpark applications by automatically providing session-scoped `spark_context` and `spark_session` fixtures. It enables users to configure the Spark environment, including setting SPARK_HOME and custom `spark_options`, directly within `pytest.ini`. The current version is 0.8.0, with an active development and release cycle.","language":"python","status":"active","last_verified":"Sun May 17","install":{"commands":["pip install pytest-spark"],"cli":null},"imports":["def test_example(spark_session):","def test_example(spark_context):"],"auth":{"required":false,"env_vars":[]},"quickstart":{"code":"import pytest\nfrom pyspark.sql import SparkSession\nfrom pyspark.sql.types import StructType, StructField, StringType, IntegerType\n\n# conftest.py (placed in your project's root or tests directory)\n@pytest.fixture(scope=\"session\")\ndef spark_session():\n    \"\"\"\n    Fixture for creating a SparkSession for testing.\n    This SparkSession is reused across all tests in the session.\n    \"\"\"\n    spark = SparkSession.builder \\\n        .master(\"local[*]\") \\\n        .appName(\"pytest-spark-session\") \\\n        .config(\"spark.driver.memory\", \"2g\") \\\n        .getOrCreate()\n    yield spark\n    spark.stop()\n\n# test_example.py (a sample test file)\ndef test_data_frame_creation(spark_session):\n    schema = StructType([\n        StructField(\"name\", StringType(), True),\n        StructField(\"age\", IntegerType(), True)\n    ])\n    data = [(\"Alice\", 1), (\"Bob\", 2)]\n    df = spark_session.createDataFrame(data, schema)\n    \n    assert df.count() == 2\n    assert df.columns == [\"name\", \"age\"]\n    assert df.collect()[0].name == \"Alice\"","lang":"python","description":"To get started, define a `spark_session` fixture in a `conftest.py` file. This fixture will automatically provide a SparkSession to your tests. Then, write tests that accept `spark_session` as an argument. You can run tests using `pytest` from your terminal.","tag":null,"tag_description":null,"last_tested":null,"results":[]},"compatibility":{"tag":null,"tag_description":null,"last_tested":"2026-05-17","installed_version":"0.8.0","pypi_latest":"0.8.0","is_stale":false,"summary":{"python_range":"3.10–3.9","success_rate":100,"avg_install_s":2.6,"avg_import_s":null,"wheel_type":"wheel"},"results":[{"runtime":"python:3.10-alpine","python_version":"3.10","os_libc":"alpine (musl)","variant":"pytest-spark","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"broken","install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":"30.7M"},{"runtime":"python:3.10-slim","python_version":"3.10","os_libc":"slim (glibc)","variant":"pytest-spark","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"broken","install_time_s":2.6,"import_time_s":null,"mem_mb":null,"disk_size":"31M"},{"runtime":"python:3.11-alpine","python_version":"3.11","os_libc":"alpine (musl)","variant":"pytest-spark","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"broken","install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":"33.6M"},{"runtime":"python:3.11-slim","python_version":"3.11","os_libc":"slim (glibc)","variant":"pytest-spark","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"broken","install_time_s":2.6,"import_time_s":null,"mem_mb":null,"disk_size":"34M"},{"runtime":"python:3.12-alpine","python_version":"3.12","os_libc":"alpine (musl)","variant":"pytest-spark","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"broken","install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":"25.3M"},{"runtime":"python:3.12-slim","python_version":"3.12","os_libc":"slim (glibc)","variant":"pytest-spark","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"broken","install_time_s":2.5,"import_time_s":null,"mem_mb":null,"disk_size":"26M"},{"runtime":"python:3.13-alpine","python_version":"3.13","os_libc":"alpine (musl)","variant":"pytest-spark","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"broken","install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":"25.0M"},{"runtime":"python:3.13-slim","python_version":"3.13","os_libc":"slim (glibc)","variant":"pytest-spark","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"broken","install_time_s":2.5,"import_time_s":null,"mem_mb":null,"disk_size":"26M"},{"runtime":"python:3.9-alpine","python_version":"3.9","os_libc":"alpine (musl)","variant":"pytest-spark","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"broken","install_time_s":null,"import_time_s":null,"mem_mb":null,"disk_size":"30.1M"},{"runtime":"python:3.9-slim","python_version":"3.9","os_libc":"slim (glibc)","variant":"pytest-spark","exit_code":0,"wheel_type":"wheel","failure_reason":null,"import_side_effects":"broken","install_time_s":3,"import_time_s":null,"mem_mb":null,"disk_size":"31M"}]}}