def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, orig_answer_text):
    """Returns tokenized answer spans that better match the annotated answer."""
    tok_answer_text = tokenizer.convert_tokens_to_string(tokenizer.tokenize(orig_answer_text))

    for new_start in range(input_start, input_end + 1):
        for new_end in range(input_end, new_start - 1, -1):
            text_span = tokenizer.convert_tokens_to_string(doc_tokens[new_start:(new_end + 1)])
            if text_span.strip() == tok_answer_text.strip():
                return (new_start, new_end)

    max_f1 = 0
    max_start = -1
    max_end = -1
    for new_start in range(input_start, input_end + 1):
        for new_end in range(input_end, new_start - 1, -1):
            text_span = tokenizer.convert_tokens_to_string(doc_tokens[new_start:(new_end + 1)])
            cur_f1 = compute_f1(tok_answer_text.strip(), text_span.strip())
            if cur_f1 > max_f1:
                max_f1 = cur_f1
                max_start = new_start
                max_end = new_end

    if max_start == -1 and max_end == -1:
        max_start = input_start
        max_end = input_end

    return (max_start, max_end)
Returns tokenized answer spans that better match the annotated answer.
xtune/src/transformers/data/processors/squad.py