def _split_examples(examples, config):
  """Split examples into a sampled and a remaining subset based on config."""
  # Only consider unique examples.
  # TODO(petershaw): Consider preserving the number of occurences for each
  # unique example to better weight sampling for computing the dataset encoding
  # codelength.
  examples = list(set([tuple(example) for example in examples]))
  if config.sample_size:
    # Sort by number of input tokens.
    examples_sorted = sorted(examples, key=lambda x: len(x[0].split()))
    examples_sample = examples_sorted[:config.sample_size]
    examples_other = examples_sorted[config.sample_size:]
  else:
    examples_sample = examples
    examples_other = []
  return examples_sample, examples_other
Split examples into a sampled and a remaining subset based on config.
language/compgen/nqg/model/induction/induction_utils.py