def construct_webqsp(self,
                       train_file,
                       test_file,
                       max_webqsp_ans = 1000):
    """Load WebQuestionsSP datasets from files.

    Args:
      train_file: filename for train data
      test_file: filename for test data
      max_webqsp_ans: max number of answers, dropped otherwise

    Returns:
      a list of train data and test data

    """
    self.max_webqsp_ans = max_webqsp_ans
    train_data, test_data = [], []
    self.ent2ent_relations = set()
    self.ent2cvt_relations = set()
    self.cvt2ent_relations = set()
    for filename, data_split in zip([train_file, test_file],
                                    [train_data, test_data]):
      all_data = json.load(gfile.GFile(self.root_dir + filename))
      for data in all_data['Questions']:
        qid = data['QuestionId']

        # Parse question and question entities.
        question = data['ProcessedQuestion']
        _, (question_token_ids, segment_ids, question_mask) = \
            self.bert_tokenizer.tokenize(question)
        question_entity = self.convert_fbid(data['Parses'][0]['TopicEntityMid'])
        if question_entity is None or question_entity not in self.entity2id:
          tf.logging.warn('Question %s illegal TopicEntityMid %s ' %
                          (qid, question_entity))
          continue
        question_entity_id = self.entity2id[question_entity]

        # Parse answers. Convert answers into a list of ids and pad with -1.
        answers = set()
        for parse in data['Parses']:
          answers.update([
              self.convert_fbid(ans['AnswerArgument'])
              if ans['AnswerType'] == 'Entity' else ans['AnswerArgument']
              for ans in parse['Answers']
          ])
        answers = list(answers)[:max_webqsp_ans]
        answer_ids = np.array(
            [self.entity2id[ans] for ans in answers if ans in self.entity2id],
            dtype=np.int32)
        if len(answer_ids) == 0:  # pylint: disable=g-explicit-length-test
          tf.logging.warn('Question %s no answers' % (qid))
          continue
        answer_ids = np.pad(
            answer_ids, (0, max_webqsp_ans - len(answer_ids)),
            mode='constant',
            constant_values=-1)

        # Parse constraints.
        constraint_entity_id = -1
        constraints = data['Parses'][0]['Constraints']
        if constraints and constraints[0]['ArgumentType'] == 'Entity':
          constraint = constraints[0]
          constraint_entity = self.convert_fbid(constraint['Argument'])
          constraint_entity_id = self.entity2id[constraint_entity]

        # Add one data point to the list.
        data_split.append(
            (question_token_ids, segment_ids, question_mask, question_entity_id,
             constraint_entity_id, answer_ids))

        # We further check if a relation is a ent2ent or ent2cvt relation.
        # Here, we simply assume that if the inferential chain is longer
        # than 2, then the intermediate entity is a cvt entity. This only
        # holds for the WebQuestionsSP dataset.
        inferential_chain = data['Parses'][0]['InferentialChain']
        if inferential_chain is None:
          continue
        inferential_chain = ['<fb:%s>' % rel for rel in inferential_chain]
        if len(inferential_chain) == 1:
          self.ent2ent_relations.add(inferential_chain[0])
        elif len(inferential_chain) == 2:
          self.ent2cvt_relations.add(inferential_chain[0])
          self.cvt2ent_relations.add(inferential_chain[1])

    tf.logging.info('WebQuestions data loaded successfully!')
    tf.logging.info('train: %d, test: %d' % (len(train_data), len(test_data)))
    return train_data, test_data
Load WebQuestionsSP datasets from files.

    Args:
      train_file: filename for train data
      test_file: filename for test data
      max_webqsp_ans: max number of answers, dropped otherwise

    Returns:
      a list of train data and test data

    
language/emql/data_loader.py