def get_recursive_chunker():
    """Get or create a recursive chunker optimized for speed."""
    global _recursive_chunker
    if _recursive_chunker is None:
        from chonkie import RecursiveLevel, RecursiveRules

        # Simpler rules for faster processing
        rules = RecursiveRules(
            [
                RecursiveLevel(delimiters=["\n\n\n"], include_delim="next"),
                RecursiveLevel(delimiters=["\n\n"], include_delim="next"),
                RecursiveLevel(delimiters=["\n"], include_delim="next"),
                RecursiveLevel(delimiters=[". ", "! ", "? "], include_delim="prev"),
            ]
        )

        _recursive_chunker = RecursiveChunker(
            tokenizer_or_token_counter=count_tokens,
            chunk_size=SAFE_CHUNK_SIZE,
            rules=rules,
            min_characters_per_chunk=100,
        )
    return _recursive_chunker
Get or create a recursive chunker optimized for speed.
backend/airweave/platform/transformers/optimized_file_chunker.py