def get_features(self, doc_tokens, qry_tokens, pos_to_sp, uid):
    """Convert list of tokens to a feature dict."""
    max_tokens_doc = self.max_seq_length - self.max_qry_length - 1
    max_tokens_qry = self.max_qry_length - 2
    doc_input_ids = self.tokenizer.convert_tokens_to_ids(
        doc_tokens[:max_tokens_doc] + ["[SEP]"])
    doc_segment_ids = [1] * len(doc_input_ids)
    doc_input_mask = [1] * len(doc_input_ids)
    qry_input_ids = self.tokenizer.convert_tokens_to_ids(
        ["[CLS]"] + qry_tokens[:max_tokens_qry] + ["[SEP]"])
    qry_segment_ids = [0] * len(qry_input_ids)
    qry_input_mask = [1] * len(qry_input_ids)
    input_ids = qry_input_ids + doc_input_ids
    input_mask = qry_input_mask + doc_input_mask
    segment_ids = qry_segment_ids + doc_segment_ids
    all_tokens = (["[CLS]"] + qry_tokens[:max_tokens_qry] + ["[SEP]"] +
                  doc_tokens[:max_tokens_doc] + ["[SEP]"])
    while len(input_ids) < self.max_seq_length:
      input_ids.append(0)
      input_mask.append(0)
      segment_ids.append(0)
    doc_start = len(qry_tokens[:max_tokens_qry]) + 2
    supporting_mask = []
    for ii in range(self.max_seq_length):
      if ii - doc_start in pos_to_sp:
        supporting_mask.append(1)
      else:
        supporting_mask.append(0)
    return {
        "unique_ids": uid,
        "input_ids": input_ids,
        "input_mask": input_mask,
        "segment_ids": segment_ids,
        "supporting_mask": supporting_mask,
    }, all_tokens, doc_start
Convert list of tokens to a feature dict.
language/labs/drkit/hotpotqa/demo.py