def process_files_with_delta():
    """
    Process files in the test directory using DataChain with delta mode.
    This demonstrates incremental processing - only new files are processed.
    """
    chain = (
        dc.read_storage("test/", update=True, delta=True, delta_on="file.path")
        .filter(C("file.path").glob("*.txt"))
        .map(file_number=extract_file_number)
        .map(content=lambda file: file.read_text())
        .map(processed_at=lambda: time.strftime("%Y-%m-%d %H:%M:%S"))
        .save(name="test_files")
    )

    # Show information about the dataset
    print(f"\nProcessed files. Total records: {chain.count()}")
    print("\nDataset versions:")
    test_dataset = dc.datasets().filter(C("name") == "test_files")

    for version in test_dataset.collect("version"):
        print(f"- Version: {version}")

    # Show the last 3 records to demonstrate the incremental processing
    print("\nLatest files processed:")
    chain.order_by("file_number", descending=True).limit(3).show()

    Process files in the test directory using DataChain with delta mode.
    This demonstrates incremental processing - only new files are processed.
    
examples/incremental_processing/delta.py