def create_data(self, train_mode):
    """Create the source, target and vocab files.

    Args:
      train_mode: string, whether we are in train, dev or test mode
    """

    # Open the 6 files.
    trainsource, traintarget, devsource, devtarget, testsource, testtarget = \
        self.open_6_files()

    # Open the raw data.
    train_dialogs = open(
        os.path.join(self._raw_data, 'train_none_original_no_cands.txt'),
        errors='ignore')
    valid_dialogs = open(
        os.path.join(self._raw_data, 'valid_none_original_no_cands.txt'),
        errors='ignore')
    filenames = [train_dialogs, valid_dialogs]

    # Copy the data to a new file.
    with open(os.path.join(self._raw_data,
                           'full_none_original_no_cands.txt'), 'w') as outfile:
      for fname in filenames:
        with fname as infile:
          outfile.write(infile.read())
    train_dialogs.close()
    valid_dialogs.close()

    # Open the big file.
    dialogs = open(
        os.path.join(self._raw_data, 'full_none_original_no_cands.txt'),
        errors='ignore')

    number_of_lines = 0
    current_dialog = ''
    dialog_list = []
    dialog_silenced = False
    # Iterate through the file and build list of dialogs separated by __eou__.
    for line in dialogs:
      if number_of_lines % 10000 == 0:
        print('problem_log: Parsed ' + str(number_of_lines) + ' lines.')

      dialog_id = line.split()[0]
      # Check if this is a refurbished line.
      if ('__SILENCE__' not in line and
          ((dialog_silenced and dialog_id == '1') or not dialog_silenced)):
        dialog_silenced = False
        number_of_lines += 1

        # Get the utterances.
        source = ' '.join(line.split('\t')[0].split()[1:])
        target = line.split('\t')[1].strip('\n')
        source = self.clean_line(source.lower())
        target = self.clean_line(target.lower())

        # Whether this is a new dialog.
        if dialog_id == '1' and current_dialog:
          dialog_list.append(current_dialog)
          current_dialog = source + '__eou__' + target + '__eou__'
        else:
          current_dialog += source + '__eou__' + target + '__eou__'
      else:
        dialog_silenced = True

      if (self.targeted_dataset_size != 0 and
          self.targeted_dataset_size < number_of_lines):
        break
    dialogs.close()

    vocabulary = collections.Counter()
    number_of_dialogs = 0
    dataset_split_counter = 0
    # Build the dataset.
    for dialog in dialog_list:
      if number_of_dialogs % 1000 == 0:
        print('problem_log: Parsed ' + str(number_of_dialogs) + ' dialogs.')

      # Check which file we should write to.
      if dataset_split_counter <= self.dataset_split['train']:
        source_file = trainsource
        target_file = traintarget
      elif dataset_split_counter <= (self.dataset_split['train'] +
                                     self.dataset_split['val']):
        source_file = devsource
        target_file = devtarget
      else:
        source_file = testsource
        target_file = testtarget

      utterances = dialog.split('__eou__')[:-1]
      i = 0
      # Loop through the dialog.
      for utterance in utterances:
        i += 1
        # Build vocabulary.
        if dataset_split_counter <= self.dataset_split['train']:
          words = utterance.split()
          for word in words:
            if word in vocabulary:
              vocabulary[word] += 1
            else:
              vocabulary[word] = 1

        # Write to files.
        if i != len(utterances):
          source_file.write(utterance + '\n')
        if i != 1:
          target_file.write(utterance + '\n')

      dataset_split_counter += 1
      number_of_dialogs += 1
      # Reset the split counter if we reached 100%.
      if dataset_split_counter == 100:
        dataset_split_counter = 0

    # Close the files.
    self.close_n_files([trainsource,
                        traintarget,
                        devsource,
                        devtarget,
                        testsource,
                        testtarget])
    # Save the vocabulary.
    self.save_vocab(vocabulary)
Create the source, target and vocab files.

    Args:
      train_mode: string, whether we are in train, dev or test mode
    
tensor2tensor/data_generators/dialog_personachat.py