def test_materialize_dataset_hadoop_config(tmpdir_factory):
    """Test that using materialize_dataset does not alter the hadoop_config"""

    path = tmpdir_factory.mktemp('data').strpath
    tmp_url = "file://" + path
    # This test does not properly check if parquet.enable.summary-metadata is restored properly with pyspark < 2.4
    spark = SparkSession.builder.getOrCreate()
    hadoop_config = spark.sparkContext._jsc.hadoopConfiguration()

    parquet_metadata_level = "COMMON_ONLY"
    parquet_row_group_check = 100

    # Set the parquet summary files and row group size check min
    hadoop_config.set('parquet.summary.metadata.level', parquet_metadata_level)
    hadoop_config.setInt('parquet.row-group.size.row.check.min', parquet_row_group_check)
    assert hadoop_config.get('parquet.summary.metadata.level') == str(parquet_metadata_level)
    assert hadoop_config.get('parquet.row-group.size.row.check.min') == str(parquet_row_group_check)

    create_test_dataset(tmp_url, range(10), spark=spark)

    assert not os.path.exists(os.path.join(path, "_metadata"))

    # Check that they are back to the original values after writing the dataset
    hadoop_config = spark.sparkContext._jsc.hadoopConfiguration()
    assert hadoop_config.get('parquet.summary.metadata.level') == str(parquet_metadata_level)
    assert hadoop_config.get('parquet.row-group.size.row.check.min') == str(parquet_row_group_check)
    # Other options should return to being unset
    assert hadoop_config.get('parquet.block.size') is None
    assert hadoop_config.get('parquet.block.size.row.check.min') is None
    spark.stop()
Test that using materialize_dataset does not alter the hadoop_config
petastorm/tests/test_end_to_end.py