def split_text(
        text: str,
        separators: List[str],
        chunk_size: int,
        chunk_overlap: int,
        length_function: Callable[[str], int]
    ) -> List[str]:

    """Split incoming text and return chunks."""
    final_chunks = []
    # Get appropriate separator to use
    separator = separators[-1]
    new_separators = []
    for i, _s in enumerate(separators):
        _separator = re.escape(_s)
        if _s == "":
            separator = _s
            break
        if re.search(_separator, text):
            separator = _s
            new_separators = separators[i + 1 :]
            break

    keep_separator = True
    _separator = re.escape(separator)
    splits = _split_text_with_regex(text, _separator, keep_separator)

    # Now go merging things, recursively splitting longer texts.
    _good_splits = []
    _separator = "" if keep_separator else separator
    for s in splits:
        if length_function(s) < chunk_size:
            _good_splits.append(s)
        else:
            if _good_splits:
                merged_text = _merge_splits(_good_splits, _separator, chunk_size, chunk_overlap, length_function)
                final_chunks.extend(merged_text)
                _good_splits = []
            if not new_separators:
                final_chunks.append(s)
            else:
                other_info = split_text(s, new_separators, chunk_size, chunk_overlap, length_function)
                final_chunks.extend(other_info)
    if _good_splits:
        merged_text = _merge_splits(_good_splits, _separator, chunk_size, chunk_overlap, length_function)
        final_chunks.extend(merged_text)
    return final_chunks
Split incoming text and return chunks.
src/aios/knowledge/data/writer.py