def retokenize(self, tokens: list[int], cont_idxs: list[int]):
        """Chat API will never respond with a word-initial space.

        If the continuation tokens begin with a word initial space, we need to
        re-tokenize with the space removed.
        """
        original_len = len(tokens)
        retokenized_continuation = cast(
            Any,
            self.tokenizer(
                self.tokenizer.decode(
                    tokens[cont_idxs[0]:cont_idxs[-1] + 1],
                ).strip(),
            )['input_ids'],
        )

        # replace the original continuation with the retokenized continuation + padding
        padding = [tokens[-1]] * (
            len(tokens) - len(tokens[:cont_idxs[0]] + retokenized_continuation)
        )
        tokens = tokens[:cont_idxs[0]] + retokenized_continuation + padding

        if len(tokens) > original_len:
            # this only happens if we were already at max seq len and the continuation got LARGER
            tokens = tokens[-original_len:]
            cont_idxs = list(
                range(
                    original_len - len(retokenized_continuation),
                    original_len,
                ),
            )
        else:
            cont_idxs = list(
                range(
                    cont_idxs[0],
                    cont_idxs[0] + len(retokenized_continuation),
                ),
            )
        return torch.tensor(tokens), torch.tensor(cont_idxs)
Chat API will never respond with a word-initial space.

        If the continuation tokens begin with a word initial space, we need to
        re-tokenize with the space removed.
        
llmfoundry/models/inference_api_wrapper/openai_causal_lm.py