Databricks Fixtures for PyTest
This creates fixtures for running tests as jobs in Databricks. This does NOT use the yield
keyword since the fixtures load the existing context natively
import pytest
@pytest.fixture
def spark():
import pyspark
from pyspark import SparkContext
from pyspark.sql import SparkSession
sc = SparkContext.getOrCreate()
spark = SparkSession()
return spark
def some_test(spark):
from pyspark.dbutils import DBUtils
dbutils = DBUtils(spark)
# test