def split_sentences(self, file_name):
        """Splits documents into sentences (if enabled)."""
        input_file_name, output_file_name = file_name
        print("Opening", input_file_name, "for sentence splitting")
        try:
            fin = open(input_file_name, 'r', encoding='utf-8')
            fout = open(output_file_name, 'w', encoding='utf-8') # Ensure utf-8 for output
        except Exception as e:
            print(f"Error opening files for sentence splitting: {e}")
            return


        encoder = Encoder(self.args)
        # Setup multiprocessing pool
        try:
             pool = multiprocessing.Pool(self.workers, initializer=encoder.initializer)
             # Assuming encoder.split exists and handles sentence splitting based on args.lang
             split_docs = pool.imap(encoder.split, fin, 32) # Use imap for memory efficiency
        except Exception as e:
             print(f"Error creating multiprocessing pool or starting imap: {e}")
             fin.close()
             fout.close()
             return

        proc_start = time.time()
        total_bytes_processed = 0
        processed_count = 0
        try:
            for i, result in enumerate(split_docs, start=1):
                # Assuming encoder.split returns (doc_string, bytes_processed)
                if isinstance(result, tuple) and len(result) == 2:
                     doc, bytes_processed = result
                     total_bytes_processed += bytes_processed
                     fout.write(doc + "\n") # Fixed newline
                     processed_count = i
                     self.print_processing_stats(i, proc_start, total_bytes_processed)
                else:
                     print(f"Warning: Unexpected result format from encoder.split: {result}")
        except Exception as e:
            print(f"Error during sentence splitting processing: {e}")
        finally:
            pool.close()
            pool.join()
            fin.close()
            fout.close()
            print(f"Finished sentence splitting for {input_file_name}. Processed {processed_count} documents.")
Splits documents into sentences (if enabled).
finetune/core/preprocess_data_conditional_xcodec_segment.py