def _clean_text_for_chunking(text: str) -> str:
    """Clean text by removing problematic Unicode characters."""
    zero_width_chars = [
        "\u200b",  # Zero-width space
        "\u200c",  # Zero-width non-joiner
        "\u200d",  # Zero-width joiner
        "\u200e",  # Left-to-right mark
        "\u200f",  # Right-to-left mark
        "\ufeff",  # Zero-width no-break space
        "\u00a0",  # Non-breaking space (replace with regular space)
    ]

    cleaned_text = text
    for char in zero_width_chars:
        if char in cleaned_text:
            count = cleaned_text.count(char)
            if count > 10:  # Only log if significant
                logger.debug(f"Removing {count} instances of Unicode {ord(char):04X} from text")
            # Replace non-breaking space with regular space instead of removing
            if char == "\u00a0":
                cleaned_text = cleaned_text.replace(char, " ")
            else:
                cleaned_text = cleaned_text.replace(char, "")

    # Also collapse multiple spaces into single space
    import re

    cleaned_text = re.sub(r"\s+", " ", cleaned_text)
    return cleaned_text
Clean text by removing problematic Unicode characters.
backend/airweave/platform/transformers/entity_field_chunker.py