def preprocess_mapper(features, lookup_table):
  """Model-specific preprocessing of features from the dataset."""
  # Truncate contexts that are too long.
  features["context"] = features["context"][:FLAGS.max_context_len]

  # Add the input lengths to the dataset ("question_len" and "context_len").
  features = dataset_utils.length_mapper(["question", "context"])(features)

  # Add the word IDs to the dataset ("question_wid" and "context_wid").
  features = dataset_utils.string_to_int_mapper(["question", "context"],
                                                mapping=lookup_table,
                                                suffix="_wid")(
                                                    features)

  # Add the character IDs to the dataset ("question_cid" and "context_cid").
  features = char_utils.token_to_char_ids_mapper(["question", "context"])(
      features)
  return features
Model-specific preprocessing of features from the dataset.
language/question_answering/decatt_docreader/models/nq_short_pipeline_model.py