def process_document(text, title, overall_regex, pattern_to_entities):
  """Process a single line from wiki.txt."""
  global document_id
  doc_obj = _preprocess_line(text, overall_regex, pattern_to_entities)
  title_obj = _preprocess_line(title, overall_regex, pattern_to_entities)
  doc_obj["id"] = document_id
  doc_obj["title"] = title_obj
  # Prepend title to the context.
  doc_obj["context"] = doc_obj["title"]["context"] + " . " + doc_obj["context"]
  extra = len(doc_obj["title"]["context"]) + 3
  for mention in doc_obj["mentions"]:
    mention["start"] += extra
  doc_obj["mentions"] = doc_obj["title"]["mentions"] + doc_obj["mentions"]
  document_id += 1
  return doc_obj
Process a single line from wiki.txt.
language/labs/drkit/metaqa/preprocessing/process_wiki.py