def generate_datasets(
    context_len, file="data/tokenized_stories_train_wikitext103.jbl", number=100, min_len=1026, trim=True
):
    """
    Generate objective set and training set

    Args:
        context_len: The maximum total input sequence length after tokenization. Sequences longer
                than this will be truncated, sequences shorter will be padded
        file: Tokenized data split into training set and objective set
        number: size of objective dataset
        min_len: minimum length of a context in objective set
        trim: If True truncate the context if it exceeds context length

    Returns:
        Generated objective set and training data


    """
    # Generate objective set and training set
    # Designate the first number (100) articles that are long enough to be used
    # as our objective set, rest (that are long enough) are training data for
    # secondary learner

    data = joblib.load(file)
    print("data loaded")
    objective_set = []
    if trim:
        for i, example in enumerate(data):
            if len(example[0]) > min_len:
                start = random.randint(0, len(example[0]) - context_len - 1)
                objective_set.append(example[0, start : start + context_len])
            if len(objective_set) >= number:
                break
        train_data = []
        for j in range(i + 1, len(data)):
            if len(data[j][0]) > min_len:
                train_data.append(data[j])
    else:
        objective_set = data[0:number]
        train_data = data[number:]

    joblib.dump(objective_set, "objective_set.jbl")
    print("objective set saved")
    return train_data, objective_set

    Generate objective set and training set

    Args:
        context_len: The maximum total input sequence length after tokenization. Sequences longer
                than this will be truncated, sequences shorter will be padded
        file: Tokenized data split into training set and objective set
        number: size of objective dataset
        min_len: minimum length of a context in objective set
        trim: If True truncate the context if it exceeds context length

    Returns:
        Generated objective set and training data


    
dpkd/transformers/examples/research_projects/information-gain-filtration/igf/igf.py