examples/tensorflow/token-classification/run_ner.py/sample_generator
def sample_generator(dataset, tokenizer, shuffle, pad_to_multiple_of=None):
    # Trim off the last partial batch if present
    if shuffle:
        sample_ordering = np.random.permutation(len(dataset))
    else:
        sample_ordering = np.arange(len(dataset))
    for sample_idx in sample_ordering:
        example = dataset[int(sample_idx)]
        # Handle dicts with proper padding and conversion to tensor.
        example = tokenizer.pad(example, return_tensors="np", pad_to_multiple_of=pad_to_multiple_of)
        if tokenizer.pad_token_id is not None:
            example["labels"][example["attention_mask"] == 0] = -100
        example = {key: tf.convert_to_tensor(arr) for key, arr in example.items()}

        yield example, example["labels"]  # TF needs some kind of labels, even if we don't use them
    return