langchain/document_loaders/pyspark_dataframe.py/PySparkDataFrameLoader/get_num_rows
class PySparkDataFrameLoader:    def get_num_rows(self) -> Tuple[int, int]:
        """Gets the amount of "feasible" rows for the DataFrame"""
        row = self.df.limit(1).collect()[0]
        estimated_row_size = sys.getsizeof(row)
        mem_info = psutil.virtual_memory()
        available_memory = mem_info.available
        max_num_rows = int(
            (available_memory / estimated_row_size) * self.fraction_of_memory
        )
        return min(max_num_rows, self.df.count()), max_num_rows