def char_token_ratio(example):
    """Compute character/token ratio of the file with tokenizer."""
    input_ids = tokenizer(example["content"], truncation=False)["input_ids"]
    ratio = len(example["content"]) / len(input_ids)
    return {"ratio": ratio}
Compute character/token ratio of the file with tokenizer.
dpkd/transformers/examples/research_projects/codeparrot/scripts/preprocessing.py