def CreateMissingTextData(data_pb, data_pbtxt_file_z, data_pbtxt_file_nnz):
  """Some cases have text and some don't. This method separates them out."""
  prefix = data_pb.prefix
  data_pb_z = util.CopyDataset(data_pb)
  data_pb_nnz = util.CopyDataset(data_pb)
  data_pb_z.name = 'flickr_zero_text'
  data_pb_nnz.name = 'flickr_non_zero_text'
  del data_pb_z.data[:]
  del data_pb_nnz.data[:]

  for tag in ['labelled', 'unlabelled']:
    # Find the proto that describes text data.
    text_data = next(d for d in data_pb.data if d.name == 'text_%s' % tag)

    # Load the text data into a sparse matrix.
    text_data_file = os.path.join(prefix, text_data.file_pattern)
    data = dh.Disk.LoadSparse(text_data_file)

    # Find cases which have non-zero words.
    numwords = np.array(data.sum(axis=1)).reshape(-1)
    nnz_indices = np.where(numwords != 0)[0]
    z_indices = np.where(numwords == 0)[0]
    indices_file = os.path.join(prefix, 'text', 'indices_%s.npz' % tag)
    np.savez(indices_file, nnz_indices=nnz_indices, z_indices=z_indices)

    text_nnz_file = os.path.join('text', 'text_nnz_2000_%s.npz' % tag)
    dh.Disk.SaveSparse(os.path.join(prefix, text_nnz_file), data[nnz_indices])
    nnz = len(nnz_indices)

    # Separate images.
    image_data = next(d for d in data_pb.data if d.name == 'image_%s' % tag)
    numdims = np.prod(image_data.dimensions)
    image_z_dir = os.path.join('image', '%s_z' % tag)
    image_nnz_dir = os.path.join('image', '%s_nnz' % tag)

    data_writer_nnz = dh.DataWriter(['combined'],
                                    os.path.join(prefix, image_nnz_dir), '1G',
                                    [numdims], datasize=nnz)
    data_writer_z = dh.DataWriter(['combined'],
                                  os.path.join(prefix, image_z_dir), '1G',
                                  [numdims], datasize=image_data.size - nnz)
    end = 0
    img_files = glob.glob(os.path.join(data_pb.prefix, image_data.file_pattern))
    for img_file in sorted(img_files):
      print img_file
      img = np.load(img_file)
      start = end
      end = start + img.shape[0]
      nw = numwords[start:end]
      zero_text_images = img[np.where(nw == 0)[0]]
      non_zero_text_images = img[np.where(nw != 0)[0]]
      data_writer_z.Submit([zero_text_images])
      data_writer_nnz.Submit([non_zero_text_images])
    num_outputs_z = data_writer_z.Commit()
    num_outputs_nnz = data_writer_nnz.Commit()
    assert num_outputs_z[0] == image_data.size - nnz
    assert num_outputs_nnz[0] == nnz

    # Make data pbtxt for the new data.
    image_z = util.CopyData(image_data)
    image_nnz = util.CopyData(image_data)
    text_nnz = util.CopyData(text_data)

    image_z.file_pattern = os.path.join(image_z_dir, 'combined-*-of-*.npy')
    image_nnz.file_pattern = os.path.join(image_nnz_dir, 'combined-*-of-*.npy')
    text_nnz.file_pattern = text_nnz_file

    image_z.size = image_data.size - nnz
    image_nnz.size = nnz
    text_nnz.size = nnz

    data_pb_z.data.extend([image_z])
    data_pb_nnz.data.extend([image_nnz, text_nnz])

  with open(data_pbtxt_file_z, 'w') as f:
    text_format.PrintMessage(data_pb_z, f)

  with open(data_pbtxt_file_nnz, 'w') as f:
    text_format.PrintMessage(data_pb_nnz, f)
Some cases have text and some don't. This method separates them out.
deepnet/examples/multimodal_dbn/setup_data.py