def create_data(self, train_mode):
    """Create the source, target and vocab files.

    Args:
      train_mode: string, whether we are in train, dev or test mode
    """

    # open the 6 files
    trainsource, traintarget, devsource, devtarget, testsource, testtarget = \
        self.open_6_files()

    conv_id = 0
    number_of_lines = 0
    dataset_split_counter = 0
    vocabulary = collections.Counter()
    # Dind all the files.
    for root, _, files in os.walk(self._raw_data_dir):
      for f in files:
        if conv_id % 100 == 0:
          print('problem_log: Parsed ' + str(conv_id) + ' files.')

        source_lines = ''
        target_lines = ''
        conv_id += 1
        dataset_split_counter += 1

        # Open one .xml file and parse it.
        with open(os.path.join(root, f), 'r', errors='ignore') as txt_file:
          words = ''
          line_id = 1

          # Parse one line.
          for line in txt_file:
            line = str(line)

            # Check if it's a new sentence.
            if line.find('<s id="') != -1:
              if words:
                # Do some cleaning.
                words = self.clean_line(words)

                # Build the vocabulary.
                if dataset_split_counter <= self.dataset_split['train']:
                  word_list = words.split()
                  for word in word_list:
                    vocabulary[word] = vocabulary.get(word, 0) + 1

                # Add the previous line.
                source_lines += words + '\n'
                if line_id != 1:
                  target_lines += words + '\n'
                line_id += 1
              words = ''

            else:
              index = line.find('<w id="')
              if index >= 0:
                line = line[index:]
                word = line[line.find('>') + 1:line.find('</w')]
                words = words + ' ' + word.replace('\t', ' ')

          # Delete the final source sentence, since it doesn't have a target.
          source_lines = '\n'.join(source_lines.split('\n')[:-2]) + '\n'

        # Save the dialog according to the dataset split.
        if dataset_split_counter <= self.dataset_split['train']:
          trainsource.write(source_lines)
          traintarget.write(target_lines)
        elif dataset_split_counter <= (self.dataset_split['train'] +
                                       self.dataset_split['val']):
          devsource.write(source_lines)
          devtarget.write(target_lines)
        else:
          testsource.write(source_lines)
          testtarget.write(target_lines)

        # Reset the split counter if we reached 100%.
        if dataset_split_counter == 100:
          dataset_split_counter = 0

        # Check if we reached the desired dataset size.
        number_of_lines += line_id
        if (self.targeted_dataset_size != 0 and
            self.targeted_dataset_size < number_of_lines):
          break
      else:
        continue
      break

    # Close the files.
    self.close_n_files([trainsource,
                        traintarget,
                        devsource,
                        devtarget,
                        testsource,
                        testtarget])
    # Save the vocabulary.
    self.save_vocab(vocabulary)
Create the source, target and vocab files.

    Args:
      train_mode: string, whether we are in train, dev or test mode
    
tensor2tensor/data_generators/dialog_opensubtitles.py