langchain/document_loaders/pyspark_dataframe.py/PySparkDataFrameLoader/__init__
class PySparkDataFrameLoader:    def __init__(
        self,
        spark_session: Optional["SparkSession"] = None,
        df: Optional[Any] = None,
        page_content_column: str = "text",
        fraction_of_memory: float = 0.1,
    ):
        """Initialize with a Spark DataFrame object."""
        try:
            from pyspark.sql import DataFrame, SparkSession
        except ImportError:
            raise ValueError(
                "pyspark is not installed. "
                "Please install it with `pip install pyspark`"
            )

        self.spark = (
            spark_session if spark_session else SparkSession.builder.getOrCreate()
        )

        if not isinstance(df, DataFrame):
            raise ValueError(
                f"Expected data_frame to be a PySpark DataFrame, got {type(df)}"
            )
        self.df = df
        self.page_content_column = page_content_column
        self.fraction_of_memory = fraction_of_memory
        self.num_rows, self.max_num_rows = self.get_num_rows()
        self.rdd_df = self.df.rdd.map(list)
        self.column_names = self.df.columns