def feature_selection_experiment(test_set):
    """
    Select top k features. Vary k from 1000 to 50000 and plot data
    """
    keys = positive.keys() + negative.keys()
    sorted_keys = sorted(keys, cmp=lambda x, y: mutual_info(x) > mutual_info(y)) # Sort descending by mutual info
    features = set()
    num_features, accuracy = [], []
    print sorted_keys[-100:]

    for k in xrange(0, 50000, 1000):
        features |= set(sorted_keys[k:k+1000])
        preprocessor = partial(reduce_features, features)
        correct = 0
        for text, label in test_set:
            correct += classify(text) == label
        num_features.append(k+1000)
        accuracy.append(correct / len(test_set))
    print negate_sequence("Is this a good idea")
    print reduce_features(features, "Is this a good idea")

    pylab.plot(num_features, accuracy)
    pylab.show()

    Select top k features. Vary k from 1000 to 50000 and plot data
    
pretrained.py