1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
|
# myurllib.py - general purpose URL handling library
#
# Copyright (C) 2007 Arthur de Jong
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
#
# The files produced as output from the software do not automatically fall
# under the copyright of the software, unless explicitly stated otherwise.
import urlparse
import re
import urllib
# The way I read RFC3986 (especially sections 3.3 and 6.2) is that these
# are all separate and valid URLs that point to the same resource.
#
# In section 6.2.2.3 only the removal of "." and ".." in paths is
# mentioned although 6.2.3 does leave some room for other normalisation.
# pattern for matching url encoded characters
_urlencpattern = re.compile('(%[0-9a-fA-F]{2})')
# characters that should not be escaped in urls
_reservedurlchars = ';/?:@&=+$,%#'
_okurlchars = '-.0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' \
'_abcdefghijklmnopqrstuvwxyz~'
def _normalize_escapes(url):
"""Ensure that escaping in the url is consistent."""
# url decode any printable normal characters
# except reserved characters with special meanings in urls
for c in _urlencpattern.findall(url):
r = chr(int(c[1:3], 16))
if r in _okurlchars:
url = url.replace(c, r)
# TODO: uppercase any escaped codes left
# TODO: make this a better performing implementation as this
# function costs about 15% of all time during deserialisation
# url encode any nonprintable or problematic characters
# (but not reserved chars)
url = ''.join(
[ (x not in _reservedurlchars and
x not in _okurlchars) and ('%%%02X' % ord(x))
or x
for x in url ] )
return url
def _urlclean(url):
"""Clean the url of uneccesary parts."""
# make escaping consistent
url = _normalize_escapes(url)
# split the url in useful parts
(scheme, netloc, path, query) = urlparse.urlsplit(url)[:4]
if ( scheme == 'http' or scheme == 'https' or scheme == 'ftp' ):
# http(s) urls should have a non-empty path
if path == '':
path = '/'
# make hostname lower case
(userpass, hostport) = urllib.splituser(netloc)
netloc = hostport.lower()
# trim trailing :
if netloc[-1:] == ':':
netloc = netloc[:-1]
if userpass is not None:
netloc = userpass+'@'+netloc
# put the url back together again (discarding fragment)
return urlparse.urlunsplit((scheme, netloc, path, query, ''))
def normalizeurl(url):
"""Return a normalized URL."""
return _urlclean(url)
|