def create_data(self, train_mode):
    """Create the source, target and vocab files.

    Args:
      train_mode: string, whether we are in train, dev or test mode
    """

    # Open the 6 files.
    trainsource, traintarget, devsource, devtarget, testsource, testtarget = \
        self.open_6_files()

    # Open the raw data.
    movie_lines = open(
        os.path.join(self._raw_data, 'movie_lines.txt'), errors='ignore')
    dialog_list = self.extract_dialog_ids()

    vocabulary = collections.Counter()
    line_dict = {}
    number_of_lines = 0
    # Iterate through file.
    for line in movie_lines:
      if number_of_lines % 10000 == 0:
        print('problem_log: Parsed ' + str(number_of_lines) + ' lines.')

      line = line.split(' +++$+++ ')
      dialog_id = line[0]
      line = line[4].lower()

      # Do some cleaning.
      line = self.clean_line(line)
      line_dict[dialog_id] = line

      number_of_lines += 1
      # Check if we reached the desired dataset size.
      if (self.targeted_dataset_size != 0 and
          self.targeted_dataset_size < number_of_lines):
        break

    counter = 0
    dataset_split_counter = 0
    # Save the actual dialogs.
    for dialog in dialog_list:
      if counter % 10000 == 0:
        print('problem_log: Saved ' +
              str(counter) + '/' + str(len(dialog_list)) + ' dialogs.')

      dataset_split_counter += 1
      i = 0
      # Save one utterance.
      for utterance in dialog:
        if (utterance != dialog[-1] and
            dialog[i + 1] != 'L211194' and
            dialog[i + 1] != 'L1045'):
          source_line = line_dict[utterance] + '\n'
          target_line = line_dict[dialog[i + 1]] + '\n'

          # Save to the files according to dataset split.
          if dataset_split_counter <= self.dataset_split['train']:
            # Build vocabulary.
            words = source_line.split()
            for word in words:
              vocabulary[word] = vocabulary.get(word, 0) + 1

            trainsource.write(source_line)
            traintarget.write(target_line)

          elif dataset_split_counter <= (self.dataset_split['train'] +
                                         self.dataset_split['val']):
            devsource.write(source_line)
            devtarget.write(target_line)
          else:
            testsource.write(source_line)
            testtarget.write(target_line)
        i += 1

      # Reset the split counter if we reached 100%.
      if dataset_split_counter == 100:
        dataset_split_counter = 0
      counter += 1

    # Close the files.
    self.close_n_files([trainsource,
                        traintarget,
                        devsource,
                        devtarget,
                        testsource,
                        testtarget])
    movie_lines.close()

    # Save the vocabulary.
    self.save_vocab(vocabulary)
Create the source, target and vocab files.

    Args:
      train_mode: string, whether we are in train, dev or test mode
    
tensor2tensor/data_generators/dialog_cornell.py