def wiki_preprocess(*,
                    common_config,
                    data_dir,
                    download_dir,
                    max_sentence_id = 30):
  """Preprocess wikipedia dump to TFDS format.

  Args:
    common_config: Configuration
    data_dir: Where to write data to
    download_dir: Where to download data to, unused but required by TFDS API
    max_sentence_id: The max sentence_id to take on each wikipedia page
  """
  logging.info('Creating wikipedia dataset builder')
  wiki_db_path = common_config.wikipedia_db_path
  builder = wiki_tfds.WikipediaText(
      wiki_db_path=wiki_db_path,
      data_dir=data_dir,
      max_sentence_id=max_sentence_id,
  )

  logging.info('Preparing wikipedia dataset')
  download_config = tfds.download.DownloadConfig(
      beam_runner=runner.FlumeRunner(),)
  builder.download_and_prepare(
      download_dir=download_dir, download_config=download_config)
Preprocess wikipedia dump to TFDS format.

  Args:
    common_config: Configuration
    data_dir: Where to write data to
    download_dir: Where to download data to, unused but required by TFDS API
    max_sentence_id: The max sentence_id to take on each wikipedia page
  
language/serene/fever_cli.py