def normalize_link(self, link):
        """Normalize a link

        >>> from scrapy.link import Link
        >>> le = BaseLinkExtractor()
        >>> l = Link('http://scrapinghub.com/some/path/../dir')
        >>> le.normalize_link(l).url
        'http://scrapinghub.com/some/dir'
        >>> l = Link('http://scrapinghub.com/some//./path/')
        >>> le.normalize_link(l).url
        'http://scrapinghub.com/some/path/'

        Files with disallowed extentions or protocols are not returned
        >>> le.normalize_link(Link('myimage.jpg')) is None
        True
        >>> le.normalize_link(Link('file:///tmp/mydoc.htm')) is None
        True
        >>> le.normalize_link(Link('http://scrapinghub.com')).url
        'http://scrapinghub.com/'

        Fragments are removed
        >>> le.normalize_link(Link('http://example.com/#something')).url
        'http://example.com/'
        >>> le.normalize_link(Link('http://example.com/#something')).fragment
        'something'
        >>> le.normalize_link(Link('http://scrapinghub.com#some fragment')).url
        'http://scrapinghub.com/'

        Ajax crawling
        >>> le.normalize_link(Link('http://example.com/#!something')).url
        'http://example.com/?_escaped_fragment_=something'
        >>> le.normalize_link(Link('http://example.com/page.html?arg=1#!something')).url
        'http://example.com/page.html?arg=1&_escaped_fragment_=something'
        """
        if len(link.url) > self.max_url_len:
            return
        parsed = urlparse(link.url)
        extention = os.path.splitext(parsed.path)[1].lower()
        if parsed.scheme not in self.allowed_schemes or \
                extention in self.ignore_extensions:
            return
        # path normalization
        path = parsed.path or '/'
        path = path if path[0] != '.' else '/' + path
        path = posixpath.normpath(path)
        if parsed.path.endswith('/') and not path.endswith('/'):
            path += '/'
        if parsed.fragment.startswith('!'):
            query = '_escaped_fragment_=%s' % parsed.fragment[1:]
            query = parsed.query + '&' + query if parsed.query else query
            parsed = parsed._replace(query=query)
        link.fragment = parsed.fragment
        if path != parsed.path or parsed.fragment:
            link.url = parsed._replace(path=path, fragment='').geturl()
        return link
Normalize a link

        >>> from scrapy.link import Link
        >>> le = BaseLinkExtractor()
        >>> l = Link('http://scrapinghub.com/some/path/../dir')
        >>> le.normalize_link(l).url
        'http://scrapinghub.com/some/dir'
        >>> l = Link('http://scrapinghub.com/some//./path/')
        >>> le.normalize_link(l).url
        'http://scrapinghub.com/some/path/'

        Files with disallowed extentions or protocols are not returned
        >>> le.normalize_link(Link('myimage.jpg')) is None
        True
        >>> le.normalize_link(Link('file:///tmp/mydoc.htm')) is None
        True
        >>> le.normalize_link(Link('http://scrapinghub.com')).url
        'http://scrapinghub.com/'

        Fragments are removed
        >>> le.normalize_link(Link('http://example.com/#something')).url
        'http://example.com/'
        >>> le.normalize_link(Link('http://example.com/#something')).fragment
        'something'
        >>> le.normalize_link(Link('http://scrapinghub.com#some fragment')).url
        'http://scrapinghub.com/'

        Ajax crawling
        >>> le.normalize_link(Link('http://example.com/#!something')).url
        'http://example.com/?_escaped_fragment_=something'
        >>> le.normalize_link(Link('http://example.com/page.html?arg=1#!something')).url
        'http://example.com/page.html?arg=1&_escaped_fragment_=something'
        
slybot/slybot/linkextractor/base.py