def process_json_file(self, file_name):
        """Processes a JSONL file, encoding documents based on arguments."""
        input_file_name, output_prefix = file_name
        print("Opening", input_file_name, "for processing")
        try:
             # Handle potential gzipped files
             if input_file_name.endswith(".gz"):
                 fin = gzip.open(input_file_name, 'rt', encoding='utf-8')
             else:
                 fin = open(input_file_name, 'r', encoding='utf-8')
        except Exception as e:
             print(f"Error opening input file {input_file_name}: {e}")
             return


        startup_start = time.time()
        encoder = Encoder(self.args)
        try:
            tokenizer = _MMSentencePieceTokenizer(self.args.tokenizer_model, vocab_extra_ids=self.args.vocab_extra_ids)
            # Initialize encoder (loads tokenizer, codectool etc.)
            encoder.initializer()
            # Pass tokenizer explicitly if not done in initializer
            Encoder.tokenizer = tokenizer
        except Exception as e:
            print(f"Error initializing tokenizer or encoder: {e}")
            fin.close()
            return


        # Determine encoding function based on args
        encode_func = None
        if self.args.order == "stage2":
            print("Using encode_codec_stage_2")
            encode_func = encoder.encode_codec_stage_2
        # MERGED: Handle both ICL and non-ICL token interleaving
        elif self.args.use_token_level_interleave or (self.args.use_audio_icl and self.args.cot):
            if self.args.use_audio_icl:
                print("Using encode_token_level_interleave (ICL-CoT mode)")
            else:
                print("Using encode_token_level_interleave (standard/CoT mode)")
            encode_func = encoder.encode_token_level_interleave
        elif self.args.order in ["textfirst", "audiofirst"]:
             print(f"Using encode_mix_text_and_codec (order: {self.args.order})")
             encode_func = encoder.encode_mix_text_and_codec
        else:
             print(f"Error: Could not determine appropriate encoder function based on args: order={self.args.order}, use_audio_icl={self.args.use_audio_icl}, cot={self.args.cot}, use_token_level_interleave={self.args.use_token_level_interleave}")
             fin.close()
             return # Exit if no valid encoder function determined


        # Setup multiprocessing pool or run in debug mode
        encoded_docs = []
        pool = None
        if not DEBUG:
            try:
                # Pass necessary class variables if initializer doesn't handle them correctly
                pool = multiprocessing.Pool(self.workers, initializer=encoder.initializer)
                encoded_docs = pool.imap(encode_func, fin, 32)
            except Exception as e:
                 print(f"Error creating multiprocessing pool or starting imap: {e}")
                 fin.close()
                 if pool: pool.close(); pool.join()
                 return
        else: # DEBUG mode
            print("Running in DEBUG mode (single process)")
            # Prepare list for debug processing
            debug_docs_list = []
            for line in fin:
                 try:
                     result = encode_func(line)
                     # Check if result is valid before appending
                     if isinstance(result, tuple) and len(result) == 3:
                          ids, lens, b_processed = result
                          if ids and lens: # Ensure ids and lens are not empty
                              debug_docs_list.append(result)
                          # else: Print debug message if needed
                     # else: Print warning about invalid result format
                 except Exception as e:
                      print(f"Error processing line in DEBUG mode: {e}")
            encoded_docs = iter(debug_docs_list) # Make it iterable like imap


        level = "document"
        if self.args.split_sentences:
            level = "sentence"

        output_bin_files = {}
        output_idx_files = {}
        builders = {}

        # Initialize builders for specified keys (usually just 'text')
        try:
            # Use tokenizer.vocab_size which should be available after initialization
            dtype = indexed_dataset.DType.optimal_dtype(Encoder.tokenizer.vocab_size)
            for key in self.args.json_keys:
                # Skip 'codec' if 'text' is present, assuming merged processing
                if key == 'codec' and "text" in self.args.json_keys:
                    print("[Info] 'codec' key specified but will be processed as part of 'text'.")
                    continue
                output_bin_files[key] = f"{output_prefix}_{key}_{level}.bin"
                output_idx_files[key] = f"{output_prefix}_{key}_{level}.idx"
                builders[key] = indexed_dataset.MMapIndexedDatasetBuilder(output_bin_files[key], dtype=dtype)
        except Exception as e:
            print(f"Error initializing IndexedDataset builders: {e}")
            fin.close()
            if pool: pool.close(); pool.join()
            return


        startup_end = time.time()
        proc_start = time.time()
        total_bytes_processed = 0
        processed_count = 0
        print(f"Time to startup processing for {input_file_name}: {startup_end - startup_start:.2f} seconds")

        try:
            for i, result in enumerate(encoded_docs, start=1):
                 # Validate result format from encode function
                 if not (isinstance(result, tuple) and len(result) == 3):
                     print(f"Warning: Invalid result format received from encoder: {type(result)}. Skipping item {i}.")
                     continue

                 doc, sentence_lens_map, bytes_processed = result
                 total_bytes_processed += bytes_processed

                 # Check if doc is empty (might happen if a sample is skipped by the encoder)
                 if not doc:
                      if DEBUG: print(f"Skipping empty document result at index {i}")
                      continue

                 for key in doc.keys():
                     if key in builders:
                          # Ensure doc[key] and sentence_lens_map[key] are valid
                          if not isinstance(doc[key], list) or not isinstance(sentence_lens_map.get(key), list):
                               print(f"Warning: Invalid data format for key '{key}' in doc/lens map at index {i}. Skipping.")
                               continue
                          # Add document if lens are provided and match structure (simple list of lengths)
                          doc_lens = sentence_lens_map.get(key)
                          if doc_lens is not None:
                               builders[key].add_document(doc[key], doc_lens)
                               processed_count = i # Update count only on successful add
                          else:
                               print(f"Warning: Missing sentence lengths for key '{key}' at index {i}. Skipping.")
                     # else: Silently ignore keys not specified in --json-keys

                 self.print_processing_stats(processed_count, proc_start, total_bytes_processed)

        except Exception as e:
            print(f"Error during document processing loop: {e}")
        finally:
            # Ensure pool is closed if it exists
            if pool:
                pool.close()
                pool.join()
            # Finalize builders
            finalized_keys = []
            for key in builders.keys():
                try:
                    print(f"Finalizing index for key '{key}'...")
                    builders[key].finalize(output_idx_files[key])
                    finalized_keys.append(key)
                except Exception as e:
                    print(f"Error finalizing builder for key '{key}': {e}")

            fin.close()
            print(f"Finished processing {input_file_name}. Processed {processed_count} documents.")
            if finalized_keys:
                 print(f"Finalized outputs for keys: {finalized_keys}")
            else:
                 print("Warning: No output builders were finalized.")
Processes a JSONL file, encoding documents based on arguments.
finetune/core/preprocess_data_conditional_xcodec_segment.py