def process_data(data: Sequence[Dict[Text, Any]], spacy_model: Any,
                 tokenizer: Any):
  """Processes fever splits.

  Given claims and accompanying labels, tokenizes the claims and annotates
  claim with mention positions corresponding to noun chunks.

  Args:
    data: List of raw FEVER samples. Each raw FEVER sample contains: claim -
      text that describes the claim to be classified. label - string label,
      REFUTES, SUPPORTS or NOT ENOUGH INFO.
    spacy_model: Spacy model object.
    tokenizer: BERT tokenizer.

  Returns:
    List of processed FEVER samples. Each sample contains:
      text_ids: dense token ids.
      text_mask: dense padding mask.
      mention_start_positions: sparse array of mention starting positions.
      mention_end_positions: sparse array of mention end positions.
      mention_mask: mention padding mask.
      target: string labels converted to 0, 1, 2.
  """
  processed_data = []
  for sample in tqdm.tqdm(data):
    new_sample = {}
    parsed_claim = spacy_model(sample['claim'])
    mention_char_spans = []
    for chunk in parsed_claim.noun_chunks:
      span_start_char = parsed_claim[chunk.start].idx
      span_last_token = parsed_claim[chunk.end - 1]
      span_end_char = span_last_token.idx + len(span_last_token.text) - 1
      mention_char_spans.append((span_start_char, span_end_char))

    _, text_ids, text_mask, mention_spans, _ = tokenization_utils.tokenize_with_mention_spans(
        tokenizer=tokenizer,
        sentence=sample['claim'],
        spans=mention_char_spans,
        max_length=FLAGS.max_length,
        add_bert_tokens=True,
    )

    new_sample['text_ids'] = text_ids
    new_sample['text_mask'] = text_mask

    mention_spans = np.array(mention_spans)
    if len(mention_spans) == 0:  # pylint: disable=g-explicit-length-test
      mention_start_positions = np.array([], dtype=np.int64)
      mention_end_positions = np.array([], dtype=np.int64)
    else:
      mention_start_positions = mention_spans[:, 0]
      mention_end_positions = mention_spans[:, 1]

    mention_start_positions = mention_start_positions[:FLAGS.max_mentions]
    mention_end_positions = mention_end_positions[:FLAGS.max_mentions]

    mention_pad_shape = (0, FLAGS.max_mentions - len(mention_start_positions))

    mention_mask = np.ones(len(mention_start_positions), dtype=np.int64)
    mention_mask = np.pad(mention_mask, mention_pad_shape, mode='constant')
    mention_start_positions = np.pad(
        mention_start_positions, mention_pad_shape, mode='constant')
    mention_end_positions = np.pad(
        mention_end_positions, mention_pad_shape, mode='constant')

    new_sample['mention_start_positions'] = mention_start_positions
    new_sample['mention_end_positions'] = mention_end_positions
    new_sample['mention_mask'] = mention_mask
    new_sample['target'] = [LABEL_DICT[sample['label']]]
    processed_data.append(new_sample)

  return processed_data
Processes fever splits.

  Given claims and accompanying labels, tokenizes the claims and annotates
  claim with mention positions corresponding to noun chunks.

  Args:
    data: List of raw FEVER samples. Each raw FEVER sample contains: claim -
      text that describes the claim to be classified. label - string label,
      REFUTES, SUPPORTS or NOT ENOUGH INFO.
    spacy_model: Spacy model object.
    tokenizer: BERT tokenizer.

  Returns:
    List of processed FEVER samples. Each sample contains:
      text_ids: dense token ids.
      text_mask: dense padding mask.
      mention_start_positions: sparse array of mention starting positions.
      mention_end_positions: sparse array of mention end positions.
      mention_mask: mention padding mask.
      target: string labels converted to 0, 1, 2.
  
language/mentionmemory/data/prepare_fever.py