def create_entry_from_json(
    json_dict: Mapping[Text, Any],
    tokenizer: tydi_tokenization_interface.TokenizerWithOffsets,
    max_passages: int,
    max_position: int,
    fail_on_invalid: bool,
    ignore_yes_no_answer: bool = True) -> Dict[Text, Any]:
  """Creates an TyDi 'entry' from the raw JSON.

  The 'TyDiEntry' dict is an intermediate format that is later converted into
  the main `TyDiExample` format.

  This function looks up the chunks of text that are candidates for the passage
  answer task, inserts special context tokens such as "[ContextId=0]", and
  creates a byte index to byte index mapping between the document plaintext
  and the concatenation of the passage candidates (these could potentially
  exclude parts of the plaintext document and also include the special tokens).

  In the returned entry, `contexts` includes only the candidate passages and
  has special tokens such as [ContextId=0] added. `span_start` and `span_end`
  are byte-wise indices into `contexts` (not the original corpus plaintext).

  Args:
    json_dict: A single JSONL line, deserialized into a dict.
    tokenizer: Used to create special marker symbols to insert into the text.
    max_passages: see FLAGS.max_passages.
    max_position: see FLAGS.max_position.
    fail_on_invalid: Immediately stop if an error is found?
    ignore_yes_no_answer: Ignore yes_no_answer type in answer's input_text.

  Returns:
    If a failure was encountered and `fail_on_invalid=False`, then returns
    an empty `dict`. Otherwise returns:
    'TyDiEntry' type: a dict-based format consumed by downstream functions:
    entry = {
        "name": str,
        "id": str,
        "language": str,
        "question": {"input_text": str},
        "answer": {
          "candidate_id": annotated_idx,
          "span_text": "",
          "span_start": -1,
          "span_end": -1,
          "input_text": "passage",
        }
        "has_correct_context": bool,
        # Includes special tokens appended.
        "contexts": str,
        # Context index to byte offset in `contexts`.
        "context_to_plaintext_offset": Dict[int, int],
        "plaintext" = json_dict["document_plaintext"]
    }
  """
  for passage_answer in json_dict["passage_answer_candidates"]:
    if (passage_answer["plaintext_start_byte"] == -1 or
        passage_answer["plaintext_end_byte"] == -1):
      # Invalid byte offset found.
      return {}

  add_candidate_types_and_positions(json_dict, max_position)

  # annotated_idx: index of the first annotated context, -1 if null.
  # annotated_min_ans: minimal answer start and end char offsets,
  #                    (-1, -1) if null.
  annotation, annotated_idx, annotated_min_ans = data.get_first_annotation(
      json_dict, max_passages)
  question = {"input_text": json_dict["question_text"]}
  answer = {
      "candidate_id": annotated_idx,
      "span_text": "",
      "span_start": -1,
      "span_end": -1,
      "input_text": "passage",
  }

  # Yes/no answers are added in the input text.
  if annotation is not None:
    assert annotation["yes_no_answer"] in ("YES", "NO", "NONE")
    if annotation["yes_no_answer"] in ("YES", "NO"):
      answer["input_text"] = annotation["yes_no_answer"].lower()

  # Add a minimal answer if one was found.
  if annotated_min_ans != (-1, -1):
    answer["input_text"] = "minimal"
    span_text = data.get_candidate_text(json_dict, annotated_idx).text

    try:
      answer["span_text"] = data.byte_slice(span_text, annotated_min_ans[0],
                                            annotated_min_ans[1])
    except UnicodeDecodeError:
      logging.error("UnicodeDecodeError for example: %s",
                    json_dict["example_id"])
      if fail_on_invalid:
        raise
      return {}
    # local (passage) byte offset
    answer["span_start"], answer["span_end"] = annotated_min_ans
    try:
      expected_answer_text = data.get_text_span(
          json_dict, {
              "plaintext_start_byte":
                  annotation["minimal_answer"]["plaintext_start_byte"],
              "plaintext_end_byte":
                  annotation["minimal_answer"]["plaintext_end_byte"],
          }).text
    except UnicodeDecodeError:
      logging.error("UnicodeDecodeError for example: %s",
                    json_dict["example_id"])
      if fail_on_invalid:
        raise
      return {}
    if expected_answer_text != answer["span_text"]:
      error_message = ("Extracted answer did not match expected answer:"
                       "'{}' vs '{}'".format(expected_answer_text,
                                             answer["span_text"]))
      if fail_on_invalid:
        raise ValueError(error_message)
      else:
        logging.warn(error_message)
        return {}

  # Add a passage answer if one was found
  elif annotation and annotation["passage_answer"]["candidate_index"] >= 0:
    if ignore_yes_no_answer:
      answer["input_text"] = "passage"
    elif annotation["yes_no_answer"] not in ("YES", "NO"):
      # If yes/no answer is added in the input text, keep it. Otherwise,
      # set the answer input text to passage.
      answer["input_text"] = "passage"
    answer["span_text"] = data.get_candidate_text(json_dict, annotated_idx).text
    answer["span_start"] = 0
    answer["span_end"] = data.byte_len(answer["span_text"])

  context_idxs = []
  context_list = []
  for idx, _ in data.candidates_iter(json_dict):
    context = {
        "id": idx,
        "type": get_candidate_type_and_position(json_dict, idx)
    }
    # Get list of all byte positions of the candidate and its plaintext.
    # Unpack `TextSpan` tuple.
    context["text_map"], context["text"] = data.get_candidate_text(
        json_dict, idx)
    if not context["text"]:
      logging.error("ERROR: Found example with empty context %d.", idx)
      if fail_on_invalid:
        raise ValueError(
            "ERROR: Found example with empty context {}.".format(idx))
      return {}
    context_idxs.append(idx)
    context_list.append(context)
    if len(context_list) >= max_passages:
      break

  # Assemble the entry to be returned.
  entry = {
      "name": json_dict["document_title"],
      "id": str(json_dict["example_id"]),
      "language": json_dict["language"],
      "question": question,
      "answer": answer,
      "has_correct_context": annotated_idx in context_idxs
  }
  all_contexts_with_tokens = []
  # `offset` is a byte offset relative to `contexts` (concatenated candidate
  # passages with special tokens added).
  offset = 0
  context_to_plaintext_offset = []
  for idx, context in zip(context_idxs, context_list):
    special_token = tokenizer.get_passage_marker(context["id"])
    all_contexts_with_tokens.append(special_token)
    context_to_plaintext_offset.append([-1] * data.byte_len(special_token))
    # Account for the special token and its trailing space (due to the join
    # operation below)
    offset += data.byte_len(special_token) + 1

    if context["id"] == annotated_idx:
      answer["span_start"] += offset
      answer["span_end"] += offset
    if context["text"]:
      all_contexts_with_tokens.append(context["text"])
      # Account for the text and its trailing space (due to the join
      # operation below)
      offset += data.byte_len(context["text"]) + 1
      context_to_plaintext_offset.append(context["text_map"])
    else:
      if fail_on_invalid:
        raise ValueError("Found example with empty context.")

  # When we join the contexts together with spaces below, we'll add an extra
  # byte to each one, so we have to account for these by adding a -1 (no
  # assigned wordpiece) index at each *boundary*. It's easier to do this here
  # than above since we don't want to accidentally add extra indices after the
  # last context.
  context_to_plaintext_offset = functools.reduce(lambda a, b: a + [-1] + b,
                                                 context_to_plaintext_offset)

  entry["contexts"] = " ".join(all_contexts_with_tokens)
  entry["context_to_plaintext_offset"] = context_to_plaintext_offset
  entry["plaintext"] = json_dict["document_plaintext"]

  if annotated_idx in context_idxs:
    try:
      expected = data.byte_slice(entry["contexts"], answer["span_start"],
                                 answer["span_end"])
    except UnicodeDecodeError:
      logging.error("UnicodeDecodeError for example: %s",
                    json_dict["example_id"])
      if fail_on_invalid:
        raise
      return {}
    # This is a sanity check to ensure that the calculated start and end
    # indices match the reported span text. If this assert fails, it is likely
    # a bug in the data preparation code above. (expected, answer["span_text"])
    if expected != answer["span_text"]:
      logging.warn("*** pruned example id: %d ***", json_dict["example_id"])
      logging.warn("*** %s, %s ***", expected, answer["span_text"])
      return {}
  return entry
Creates an TyDi 'entry' from the raw JSON.

  The 'TyDiEntry' dict is an intermediate format that is later converted into
  the main `TyDiExample` format.

  This function looks up the chunks of text that are candidates for the passage
  answer task, inserts special context tokens such as "[ContextId=0]", and
  creates a byte index to byte index mapping between the document plaintext
  and the concatenation of the passage candidates (these could potentially
  exclude parts of the plaintext document and also include the special tokens).

  In the returned entry, `contexts` includes only the candidate passages and
  has special tokens such as [ContextId=0] added. `span_start` and `span_end`
  are byte-wise indices into `contexts` (not the original corpus plaintext).

  Args:
    json_dict: A single JSONL line, deserialized into a dict.
    tokenizer: Used to create special marker symbols to insert into the text.
    max_passages: see FLAGS.max_passages.
    max_position: see FLAGS.max_position.
    fail_on_invalid: Immediately stop if an error is found?
    ignore_yes_no_answer: Ignore yes_no_answer type in answer's input_text.

  Returns:
    If a failure was encountered and `fail_on_invalid=False`, then returns
    an empty `dict`. Otherwise returns:
    'TyDiEntry' type: a dict-based format consumed by downstream functions:
    entry = {
        "name": str,
        "id": str,
        "language": str,
        "question": {"input_text": str},
        "answer": {
          "candidate_id": annotated_idx,
          "span_text": "",
          "span_start": -1,
          "span_end": -1,
          "input_text": "passage",
        }
        "has_correct_context": bool,
        # Includes special tokens appended.
        "contexts": str,
        # Context index to byte offset in `contexts`.
        "context_to_plaintext_offset": Dict[int, int],
        "plaintext" = json_dict["document_plaintext"]
    }
  
language/canine/tydiqa/preproc.py