def convert_single_example_record(example, max_seq_length, tokenizer):
  """Convert single record example to features."""

  example_list = convert_example_to_cands(example)
  feature_list = []
  for struct_example in example_list:
    text = struct_example.text
    completions = struct_example.completions
    text = tokenizer.tokenize(text)
    completions = [tokenizer.tokenize(c) for c in completions]
    context_input_ids = tokenizer.convert_tokens_to_ids(text)
    comp_input_ids = [tokenizer.convert_tokens_to_ids(c) for c in completions]

    # cut size and pad
    context_input_ids = context_input_ids[:max_seq_length]
    context_input_ids += [0] * (max_seq_length - len(context_input_ids))
    max_target_len = 100
    comp_input_ids = [
        c[:max_target_len] + ([0] * (max_target_len - len(c)))
        for c in comp_input_ids
    ]

    feature = Example(context_input_ids, comp_input_ids)
    feature_list.append(feature)
  return feature_list
Convert single record example to features.
language/conpono/evals/model_builder.py