def generate(
        self,
        prompt: Union[Union[str, tuple[str]], list[Union[str, tuple[str]]]],
        adapter: Optional[str] = None,
        max_gen_len: int = 128,
        use_io_binding: bool = False,
        cache_type: str = "dynamic",
        max_cache_len: int = 1024,
        cache_backend: str = "ort",
    ) -> Union[str, list[str]]:
        """Generate text from the model given a prompt.

        :param prompt: The prompt to generate text from. Can be a string/string-tuples or a list.
        :param adapter: The adapter to use for generation. If None, the default adapter is used.
        :param max_gen_len: The maximum length of the generated text.
        :param use_io_binding: Whether to use IO binding for the generation.
        :param cache_type: The type of cache to use for the generation. Can be "static" or "dynamic".
        :param max_cache_len: The maximum length of the cache for static cache.
        :param cache_backend: The backend to use for static cache. Can be "ort" or "torch".
        :return: The generated text.
        """
        if use_io_binding and self.execution_provider == "QNNExecutionProvider":
            raise ValueError("QNNExecutionProvider does not support IO binding.")

        # get session, template and adapter inputs
        # if adapter mode is inputs, the adapter input is dictionary of adapter weights
        session, template, adapter_inputs = self.get_adapter(adapter, use_io_binding)
        io_binding = session.io_binding() if use_io_binding else None

        # get the inputs for prompt processing
        inputs, cache = self.get_initial_inputs(
            prompt, template, use_io_binding, cache_type, max_cache_len, cache_backend
        )
        inputs = {**inputs, **(adapter_inputs or {})}

        generated_tokens = inputs["input_ids"].numpy() if use_io_binding else inputs["input_ids"].copy()
        batch_size, prompt_length = generated_tokens.shape
        has_eos = np.zeros(batch_size, dtype=bool)

        # buffers to keep numpy copy of model inputs, don't want to keep going back and forth between OrtValue and numpy
        np_buffers = {
            "attention_mask": inputs["attention_mask"].numpy() if use_io_binding else inputs["attention_mask"].copy()
        }
        if self.use_position_ids:
            np_buffers["position_ids"] = (
                np_buffers["attention_mask"]
                .sum(axis=1)
                .reshape(batch_size, 1)
                .astype(self.input_info["position_ids"]["dtype"])
            )

        for idx in tqdm(range(max_gen_len)):
            used_inputs = {k: v for k, v in inputs.items() if k in self.input_info}
            if use_io_binding:
                if idx < 2:
                    # need to bind logits twice, once for prompt processing and once for token generation
                    # shapes: (batch_size, prompt_length, vocab_size) and (batch_size, 1, vocab_size)
                    io_binding.bind_output("logits", self.device, self.device_id)
                for k, v in used_inputs.items():
                    io_binding.bind_ortvalue_input(k, v)
                cache.bind_kv_io(io_binding)

                io_binding.synchronize_inputs()
                session.run_with_iobinding(io_binding)
                io_binding.synchronize_outputs()

                outputs = io_binding.get_outputs()
                logits = outputs[0].numpy()
            else:
                outputs = session.run(None, {**used_inputs, **cache.get_kv_inputs()})
                logits = outputs[0]

            # Decide the next token using your preferred sampling strategy.
            # Sample with argmax (greedy search)
            if idx == 0:
                # logits for the first token are the logits at the last token of the prompt
                next_token_logits = np.take_along_axis(
                    logits, np_buffers["attention_mask"].sum(axis=1).reshape(batch_size, 1, 1) - 1, axis=1
                ).reshape(batch_size, -1)
            else:
                next_token_logits = logits[:, -1, :]
            next_tokens = np.argmax(next_token_logits, axis=-1)

            # Check if we previously reached EOS token id or if generated token id is EOS token id
            has_eos = has_eos | next_tokens == self.tokenizer.eos_token_id

            # Determine which new tokens to add to list of all token ids
            # Add EOS token ids for batch entries that ended early
            # (ragged batching scenario where some batch entries ended early and some haven't)
            tokens_to_add = np.where(~has_eos, next_tokens, self.tokenizer.eos_token_id).reshape(batch_size, 1)
            generated_tokens = np.concatenate([generated_tokens, tokens_to_add], axis=-1)

            # Return early if all batch entries have reached EOS token id
            if np.all(has_eos):
                break

            # Update inputs for next iteration
            if use_io_binding:
                if idx == 0:
                    inputs["input_ids"] = OrtValue.ortvalue_from_numpy(tokens_to_add, self.device, self.device_id)
                    if self.use_position_ids:
                        inputs["position_ids"] = OrtValue.ortvalue_from_numpy(
                            np_buffers["position_ids"], self.device, self.device_id
                        )
                else:
                    inputs["input_ids"].update_inplace(tokens_to_add)
                    if self.use_position_ids:
                        np_buffers["position_ids"] += 1
                        inputs["position_ids"].update_inplace(np_buffers["position_ids"])
            else:
                inputs["input_ids"] = tokens_to_add
                if self.use_position_ids:
                    if idx == 0:
                        inputs["position_ids"] = np_buffers["position_ids"]
                    else:
                        inputs["position_ids"] += 1

            # NOTE: we could use !has_eos instead of 1s in the attention updates but that is not necessary
            # since we don't care about subsequent tokens after EOS token id
            # NOTE: the attention over the past is technically not correct for gqa since gqa inserts the past
            # at seqlen_k instead of appending to the end of the past
            # it still works since the gqa node only cares about the sum of the attention mask
            if cache_type == "static":
                if idx == 0 and not isinstance(cache, GQASharedCache):
                    # past is now max_cache_len and there is only one token in the input
                    # static shape: (batch_size, max_cache_len + 1, num_heads, head_dim)
                    # GQA already has static shape: (batch_size, max_cache_len, num_heads, head_dim)
                    np_buffers["attention_mask"] = np.concatenate(
                        [
                            np_buffers["attention_mask"],
                            np.zeros((batch_size, cache.max_cache_len - prompt_length + 1), dtype=np.int32),
                        ],
                        1,
                    )
                elif idx > 0:
                    # previous token becomes past
                    np_buffers["attention_mask"][:, prompt_length + idx - 1] = 1
                # always attend to the last token
                np_buffers["attention_mask"][:, -1] = 1
            else:
                np_buffers["attention_mask"] = np.concatenate(
                    [np_buffers["attention_mask"], np.ones((batch_size, 1), dtype=np.int32)], 1
                )

            if not use_io_binding:
                inputs["attention_mask"] = np_buffers["attention_mask"]
            elif cache_type == "dynamic" or (idx == 0 and not isinstance(cache, GQASharedCache)):
                # only update attention mask for dynamic cache or first token for static cache (non-GQA)
                inputs["attention_mask"] = OrtValue.ortvalue_from_numpy(
                    np_buffers["attention_mask"], self.device, self.device_id
                )
            else:
                # GQA, or static during token generation
                inputs["attention_mask"].update_inplace(np_buffers["attention_mask"])

            if self.use_past_seq_len:
                past_seq_len = (np_buffers["attention_mask"].sum(-1, keepdims=True) - 1).astype(np.int32)
                total_seq_len = np.array(np_buffers["attention_mask"].shape[1], dtype=np.int32)
                if use_io_binding:
                    inputs["past_seq_len"].update_inplace(past_seq_len)
                    inputs["total_seq_len"].update_inplace(total_seq_len)
                else:
                    inputs["past_seq_len"] = past_seq_len
                    inputs["total_seq_len"] = total_seq_len

            # update cache
            cache.update(outputs[1:])
        if use_io_binding:
            io_binding.clear_binding_inputs()
            io_binding.clear_binding_outputs()

        decoded_text = self.tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
        if isinstance(prompt, (str, tuple)):
            return decoded_text[0]
        return decoded_text
Generate text from the model given a prompt.

        :param prompt: The prompt to generate text from. Can be a string/string-tuples or a list.
        :param adapter: The adapter to use for generation. If None, the default adapter is used.
        :param max_gen_len: The maximum length of the generated text.
        :param use_io_binding: Whether to use IO binding for the generation.
        :param cache_type: The type of cache to use for the generation. Can be "static" or "dynamic".
        :param max_cache_len: The maximum length of the cache for static cache.
        :param cache_backend: The backend to use for static cache. Can be "ort" or "torch".
        :return: The generated text.
        
examples/utils/generator.py