def __init__(self, max_url_len=2083, ignore_extensions=_ignored_exts,
                 allowed_schemes=ALLOWED_SCHEMES):
        """Creates a new LinkExtractor

        The defaults are a good guess for the first time crawl. After that, we
        expect that they can be learned.
        """
        self.max_url_len = max_url_len
        self.ignore_extensions = ignore_extensions
        self.allowed_schemes = allowed_schemes
Creates a new LinkExtractor

        The defaults are a good guess for the first time crawl. After that, we
        expect that they can be learned.
        
slybot/slybot/linkextractor/base.py