def preprocess(
    *,
    common_config,
    scrape_type,
    data_dir,
    download_dir):
  """Preprocess the fever data to the TFDS Fever data.

  Args:
    common_config: Common configuration from config.Config
    scrape_type: Which scrape to use, drqa/lucene/ukp, in training
    data_dir: Where to write data to
    download_dir: Where to download data to, unused but required by TFDS API
  """
  logging.info('Creating fever dataset builder')
  text_matcher_params_path = common_config.text_matcher_params
  fever_train_path = common_config.fever_train
  fever_dev_path = common_config.fever_dev
  fever_test_path = common_config.fever_test

  ukp_docs_train = common_config.ukp_docs_train
  ukp_docs_dev = common_config.ukp_docs_dev
  ukp_docs_test = common_config.ukp_docs_test
  builder = fever_tfds.FeverEvidence(
      wiki_db_path=common_config.wikipedia_db_path,
      text_matcher_params_path=text_matcher_params_path,
      fever_train_path=fever_train_path,
      fever_dev_path=fever_dev_path,
      fever_test_path=fever_test_path,
      drqa_db_path=common_config.drqa_scrape_db_path,
      lucene_db_path=common_config.lucene_scrape_db_path,
      data_dir=data_dir,
      n_similar_negatives=common_config.n_similar_negatives,
      n_background_negatives=common_config.n_background_negatives,
      ukp_docs_train=ukp_docs_train,
      ukp_docs_dev=ukp_docs_dev,
      ukp_docs_test=ukp_docs_test,
      train_scrape_type=scrape_type,
      title_in_scoring=common_config.title_in_scoring,
      n_inference_candidates=common_config.n_inference_candidates,
      include_not_enough_info=common_config.include_not_enough_info,
      n_inference_documents=common_config.n_inference_documents,
      max_inference_sentence_id=common_config.max_inference_sentence_id,
  )

  logging.info('Preparing fever evidence dataset')
  beam_runner = direct_runner.DirectRunner()
  download_config = tfds.download.DownloadConfig(beam_runner=beam_runner,)
  builder.download_and_prepare(
      download_dir=download_dir, download_config=download_config)
Preprocess the fever data to the TFDS Fever data.

  Args:
    common_config: Common configuration from config.Config
    scrape_type: Which scrape to use, drqa/lucene/ukp, in training
    data_dir: Where to write data to
    download_dir: Where to download data to, unused but required by TFDS API
  
language/serene/fever_cli.py