def get_fresh_cache(
        self,
        batch_size: int,
        use_io_binding: bool,
        cache_type: str,
        max_cache_len: int,
        cache_backend: str,
    ) -> Union["Cache", "IOBoundCache"]:
        """Get a fresh cache for the model."""
        # determine cache class
        cache_class = None
        if cache_type == "static":
            if self.attn_type == "gqa" and use_io_binding:
                cache_class = GQASharedCache
            elif self.attn_type == "gqa":
                # TODO(jambayk): implement generic GQA static cache if needed
                # it is not the same as using the current static cache since GQA inserts
                # new kv at seqlen_k instead of appending to the end of the past
                raise ValueError("GQA model only supports static cache with IO binding.")
            elif use_io_binding:
                cache_class = StaticIOBoundCache
            else:
                cache_class = StaticCache
        elif cache_type == "dynamic":
            if use_io_binding:
                cache_class = DynamicIOBoundCache
            else:
                cache_class = DynamicCache
        else:
            raise ValueError(f"Invalid cache type: {cache_type}")

        kwargs = {"batch_size": batch_size}
        if use_io_binding:
            kwargs["device"] = self.device
            kwargs["device_id"] = self.device_id
            if cache_type == "static":
                kwargs["backend"] = cache_backend
        if cache_type == "static":
            kwargs["max_cache_len"] = max_cache_len

        return cache_class(**kwargs, **self.cache_info)
Get a fresh cache for the model.
examples/utils/generator.py