def test_split_prefix_preprocess_text_long_text_many_chunks(self, mock_preprocess):
        """Check long content chunking with small split length."""
        long_text = "word " * 100  # 100 repetitions -> 100 words
        content = [long_text]
        preprocessor = CLIPPreprocessor()
        preprocessing_config = TextPreprocessingConfig(
            should_chunk=True,
            chunk_config=TextChunkConfig(split_method="word", split_length=2, split_overlap=0),
            text_prefix="prefix: "
        )

        results = split_prefix_preprocess_text(content, preprocessor, preprocessing_config)

        # Expecting 100/2 = 50 chunks (without overlap)
        self.assertEqual(len(results), 1)
        chunks = results[0]
        self.assertEqual(len(chunks), 50)

        for chunk in chunks:
            text, tensor = chunk
            self.assertFalse(text.startswith("prefix: "))
            self.assertEqual(text, " ".join(text.split()))
            self.assertIsInstance(tensor, torch.Tensor)

        mock_preprocess.assert_called()
Check long content chunking with small split length.
tests/unit_tests/marqo/inference/media_download_and_preprocess/test_chunk_download_preprocess_text_content.py