def validate_columns_in_table(
    required_columns: list,
    optional_columns: list,
    table_name: str,
    method: str,
    cursor: Optional['Cursor'] = None,
    spark: Optional['SparkSession'] = None,
) -> bool:
    """Validate that required and optional columns exist in the Delta table."""
    try:
        result = run_query(
            f'SHOW COLUMNS IN {table_name}',
            method,
            cursor,
            spark,
        )

        # Get the actual column names
        assert result
        actual_columns = [row.asDict()['col_name'] for row in result]

        missing_required = set(required_columns) - set(actual_columns)
        allowed_columns = set(required_columns + optional_columns)
        extra_columns = set(actual_columns) - allowed_columns

        if missing_required:
            logger.error(f'Missing required columns: {missing_required}')
            return False
        if extra_columns:
            logger.warning(f'Extra columns found: {extra_columns}')
            return False

        logger.info(
            f'Table {table_name} contains the required and optional columns.',
        )
        return True
    except Exception as e:
        logger.error(f'Error validating columns in table {table_name}: {e}')
        return False
Validate that required and optional columns exist in the Delta table.
llmfoundry/command_utils/data_prep/convert_delta_to_contrastive_mds.py