def generate_examples(base_path, hps, is_train):
  """Read data.

  Args:
      base_path: base path to the files. one instance per line.
      hps: hyperparameters
      is_train: Is training or not.
  Returns:
      A list of the instances.
  """

  src_lst, tgt_lst = [], []
  neighbor_lst, neighbor_binary_lst, neighbor_id_lst = [], [], []
  with tf.gfile.Open("%s/src" % base_path, "r") as fin:
    for line in fin:
      src = line.strip().decode("utf-8").split()
      # src = line.strip().split()
      src_lst.append(src)
  with tf.gfile.Open("%s/tgt" % base_path, "r") as fin:
    for line in fin:
      tgt = line.strip().decode("utf-8").split()
      # tgt = line.strip().split()
      tgt_lst.append(tgt)
  if hps.random_neighbor:
    with tf.gfile.Open("%s/random_neighbor_id" % base_path, "r") as fin:
      for line in fin:
        ids = [int(x) for x in line.strip().split(" ")]
        neighbor_id_lst.append(ids)
  else:
    if hps.use_cluster:
      path = "%s/cluster_id" % base_path
    else:
      path = "%s/neighbor_id" % base_path
    with tf.gfile.Open(path, "r") as fin:
      for line in fin:
        ids = [int(x) for x in line.strip().split(" ")]
        neighbor_id_lst.append(ids)
  with tf.gfile.Open(
      "%s/neighbor.cleaned" % (base_path), "r") as fin:
    for line in fin:
      line = line.strip()
      neighbors = line.decode("utf-8").split(SEPARATOR)
      # neighbors = line.split(SEPARATOR)
      neighbor = [x.strip() for x in neighbors]
      if hps.sample_neighbor and is_train:
        size = min(3, len(neighbors))
        neighbor = neighbors[sample_neighbor(size=size)].split()
      else:
        neighbor = neighbors[0].split()
      # neighbor = line.strip().split()
      neighbor_lst.append(neighbor)
  if hps.binary_neighbor:
    with tf.gfile.Open(
        "%s/neighbor.binary" % (base_path), "r") as fin:
      for line in fin:
        line = line.strip()
        neighbors = line.split(SEPARATOR)
        neighbor = [x.strip() for x in neighbors]
        if hps.sample_neighbor and is_train:
          assert False
          size = min(3, len(neighbors))
          neighbor = neighbors[sample_neighbor(size=size)].split()
        else:
          neighbor = [int(x) for x in neighbors[0].split()]
        # neighbor = line.strip().split()
        neighbor_binary_lst.append(neighbor)
  else:
    neighbor_binary_lst = [[0] * len(x) for x in tgt_lst]

  assert len(src_lst) == len(tgt_lst)
  assert len(src_lst) == len(neighbor_id_lst)
  assert len(src_lst) == len(neighbor_lst)
  assert len(src_lst) == len(neighbor_binary_lst)
  examples = []
  for s, t, n_id, n, nb in zip(
      src_lst,
      tgt_lst,
      neighbor_id_lst,
      neighbor_lst,
      neighbor_binary_lst):
    examples.append(
        {"src": s, "tgt": t, "neighbor_ids": n_id, "neighbor": n,
         "neighbor_binary": nb})
  return examples
Read data.

  Args:
      base_path: base path to the files. one instance per line.
      hps: hyperparameters
      is_train: Is training or not.
  Returns:
      A list of the instances.
  
language/labs/exemplar_decoding/utils/data.py