def chunk_document(
    doc_paths: List[str],
    chunk_size: int,
    chunk_step: int,
    extensions: Optional[List[str]] = None,
) -> Tuple[int, List[str], List[Dict[str, str]], Dict[str, int]]:
    """
    Split documents into chunks
    :param doc_paths: the paths of the documents
    :param chunk_size: the size of the chunk
    :param chunk_step: the step size of the chunk
    """
    texts = []
    metadata_list = []
    file_count = 0
    chunk_id_to_index = dict()

    enc = tiktoken.encoding_for_model("gpt-3.5-turbo")

    # traverse all files under dir
    print("Split documents into chunks...")
    for doc_path in doc_paths:
        for root, dirs, files in os.walk(doc_path):
            for name in files:
                extension = name.split(".")[-1]
                if extensions is not None and extension not in extensions:
                    continue
                f = os.path.join(root, name)
                print(f"Reading {f}")
                try:
                    title, content = text_parser(f)
                    file_count += 1
                    if file_count % 100 == 0:
                        print(f"{file_count} files read.")

                    if len(content) == 0:
                        continue

                    chunks = chunk_str_overlap(
                        content.strip(),
                        num_tokens=chunk_size,
                        step_tokens=chunk_step,
                        separator="\n",
                        encoding=enc,
                    )
                    source = os.path.sep.join(f.split(os.path.sep)[4:])
                    for i in range(len(chunks)):
                        # custom metadata if needed
                        metadata = {
                            "source": source,
                            "title": title,
                            "chunk_id": i,
                        }
                        chunk_id_to_index[f"{source}_{i}"] = len(texts) + i
                        metadata_list.append(metadata)
                    texts.extend(chunks)
                except Exception as e:
                    print(f"Error encountered when reading {f}: {traceback.format_exc()} {e}")
    return file_count, texts, metadata_list, chunk_id_to_index

    Split documents into chunks
    :param doc_paths: the paths of the documents
    :param chunk_size: the size of the chunk
    :param chunk_step: the step size of the chunk
    
scripts/document_indexer.py