def generate_pages(xml_file):
  """Generates pages from Wikipedia XML file as dictionaries."""
  context = ElementTree.iterparse(xml_file, events=("start", "end"))
  context = iter(context)
  _, root = next(context)
  fields = {}
  depth = -1
  for event, element in context:
    ## Top-level logic
    if event == "start" and has_tag(element, "page"):
      fields = {}
      depth = 0
    elif event == "end" and has_tag(element, "page"):
      if validate_fields(fields):
        yield fields
      root.clear()  # Prevents memory issues.
    elif event == "start":
      depth += 1
    elif event == "end":
      depth -= 1
    ## Fields
    if event == "end" and has_tag(element, "title"):
      fields["title"] = element.text
    elif event == "end" and has_tag(element, "text"):
      fields["text"] = element.text
    elif event == "end" and has_tag(element, "redirect"):
      fields["redirect"] = element.attrib["title"]
    elif event == "end" and has_tag(element, "ns"):
      fields["ns"] = element.text
    # Using depth to ensure we get only the top-level page id, and not some
    # other id (like a revision id).
    elif event == "end" and has_tag(element, "id") and depth == 0:
      fields["id"] = int(element.text)
Generates pages from Wikipedia XML file as dictionaries.
language/fruit/wiki_utils.py