def preprocess_mapper(raw_text, params, lookup_table, vocab):
  """Model-specific preprocessing of features from the dataset."""
  features = dict(input_type=datasets.DatasetTypes.GUIDED)

  splits = tf.strings.split([raw_text], "\t").values
  question, answer, text = splits[1], splits[2], splits[3]

  text = text_utils.build_text_inputs(
      text=text,
      length=params["caption_length"],
      lookup_table=lookup_table,
      segment_id=0,
      start_token=vocab.CLS,
      end_token=vocab.SEP)
  assert isinstance(text, text_utils.TextInputs)

  features["token_inputs"] = text_utils.TextInputs(
      token_ids=text.token_ids[:-1],
      mask=text.mask[:-1],
      segment_ids=text.segment_ids[:-1],
      positions=text.positions[:-1])

  features["token_outputs"] = text_utils.TextOutputs(
      token_ids=text.token_ids[1:], mask=text.mask[1:])

  if params.get("conditional_decoding"):
    features["condition_inputs"] = text_utils.build_planner_inputs(
        question=question,
        answer=answer,
        length=params["condition_length"],
        lookup_table=lookup_table)

  # Add dummy inputs for standardization for multi-tasking.
  for k, v in datasets.footprint(params).items():
    if k not in features:
      features[k] = v

  return features
Model-specific preprocessing of features from the dataset.
language/capwap/datasets/text_dataset.py