1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
# myurllib.py - general purpose URL handling library
#
# Copyright (C) 2007, 2011 Arthur de Jong
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
#
# The files produced as output from the software do not automatically fall
# under the copyright of the software, unless explicitly stated otherwise.
import re
import urllib
import urlparse
# The way I read RFC3986 (especially sections 3.3 and 6.2) is that these
# are all separate and valid URLs that point to the same resource.
#
# In section 6.2.2.3 only the removal of "." and ".." in paths is
# mentioned although 6.2.3 does leave some room for other normalisation.
# pattern for matching URL-encoded characters
_urlencpattern = re.compile('(%[0-9a-fA-F]{2})')
# characters that should be unescaped in URLs
_okurlchars = set('-.0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' \
'_abcdefghijklmnopqrstuvwxyz~')
# pattern for matching characters that should be escaped
_urlprobpattern = re.compile('([^-;/?:@&=+$,%#.0123456789' \
'ABCDEFGHIJKLMNOPQRSTUVWXYZ_' \
'abcdefghijklmnopqrstuvwxyz~])')
# pattern for double slashes
_doubleslashpattern = re.compile('//+')
# pattern for leading dots
_leadingdotpattern = re.compile('^(/\.\.)*')
def _unescape_printable(match):
"""Helper function for _normalize_escapes() to perform the expansion of
html entity refs that are normal printable (but not reserver)
characters."""
# unescape the character
r = chr(int(match.group(1)[1:3], 16))
if r in _okurlchars:
return r
# transform remaining escapes to uppercase
return match.group(1).upper()
def _normalize_escapes(url):
"""Ensure that escaping in the url is consistent. Any reserved characters
are left alone. Any characters that are printable but are escaped are
unescaped. Any non-printable characters are escaped."""
# url decode any printable normal characters (this leaves us with a string
# with as much stuff unquoted as # possible)
url = _urlencpattern.sub(_unescape_printable, url)
# url encode any nonprintable or problematic characters (but not reserved
# characters) so we're left with a string with everything that needs to be
# quoted as such
url = _urlprobpattern.sub(lambda x: '%%%02X' % ord(x.group(1)), url)
return url
def _urlclean(url):
"""Clean the url of uneccesary parts."""
# make escaping consistent
url = _normalize_escapes(url)
# split the url in useful parts
(scheme, netloc, path, query, fragment) = urlparse.urlsplit(url)
# remove any leading /../ parts
if scheme in ('http', 'https'):
path = _leadingdotpattern.sub('', path)
if scheme in ('http', 'https', 'ftp'):
# http(s) urls should have a non-empty path
if path == '':
path = '/'
# make hostname lower case
(userpass, hostport) = urllib.splituser(netloc)
(host, port) = urllib.splitport(hostport)
# remove default port
if scheme == 'http' and str(port) == '80':
hostport = host
elif scheme == 'https' and str(port) == '443':
hostport = host
netloc = hostport.lower()
# trim trailing :
if netloc[-1:] == ':':
netloc = netloc[:-1]
if userpass is not None:
netloc = userpass + '@' + netloc
# get rid of double slashes in some paths
if scheme == 'file':
path = _doubleslashpattern.sub('/', path)
# put the url back together again
return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
def normalizeurl(url):
"""Return a normalized URL."""
return _urlclean(url)
|