1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
|
# http.py - handle urls with a http scheme
#
# Copyright (C) 1998, 1999 Albert Hopkins (marduk)
# Copyright (C) 2002 Mike W. Meyer
# Copyright (C) 2005 Arthur de Jong
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
"""This module defines the functions needed for filling in information in Link
objects for urls using the http scheme."""
import config
import debugio
import string
import httplib
import urllib
import time
import urlparse
import base64
import socket
def fetch(link, acceptedtypes):
"""Open connection to url and report information given by GET command."""
# TODO: HTTP connection pooling?
# split netloc in user:pass part and host:port part
(userpass,netloc) = urllib.splituser(link.netloc)
host = urllib.splitport(netloc)[0]
# check which host to connect to (if using proxies)
if config.PROXIES and config.PROXIES.has_key(link.scheme):
# pass the complete url in the request, connecting to the proxy
# TODO: implement proxy authentication
path = urlparse.urlunsplit((link.scheme,netloc,link.path,link.query,""))
netloc = urlparse.urlsplit(config.PROXIES[link.scheme])[1]
else:
# otherwise direct connect to the server with partial url
path = urlparse.urlunsplit(("","",link.path,link.query,""))
conn=None
try:
try:
# create the connection
if link.scheme == "http":
conn=httplib.HTTPConnection(netloc)
elif link.scheme == "https":
conn=httplib.HTTPSConnection(netloc)
# the requests adds a correct host header for us
conn.putrequest("GET", path)
if userpass is not None:
(user, passwd) = urllib.splitpasswd(userpass)
conn.putheader("Authorization", "Basic "+string.strip(base64.encodestring(user + ":" + passwd)))
conn.putheader("User-Agent","webcheck %s" % config.VERSION)
conn.endheaders()
# wait for the response
response = conn.getresponse()
debugio.debug("schemes.http.fetch(): HTTP response: %s %s" % (response.status, response.reason))
# retrieve some information from the headers
try:
link.mimetype = response.msg.gettype()
debugio.debug("schemes.http.fetch(): mimetype: %s" % str(link.mimetype))
except AttributeError:
pass
try:
link.size = int(response.getheader("Content-length"))
debugio.debug("schemes.http.fetch(): size: %s" % str(link.size))
except (KeyError, TypeError):
pass
try:
link.mtime = time.mktime(response.msg.getdate("Last-Modified"))
debugio.debug("schemes.http.fetch(): mtime: %s" % time.strftime("%c",time.localtime(link.mtime)))
except (OverflowError, TypeError, ValueError):
pass
# handle redirects
# 301=moved permanently, 302=found, 303=see other, 307=temporary redirect
if response.status == 301 or response.status == 302 or response.status == 303 or response.status == 307:
# consider a 301 (moved permanently) a problem
if response.status == 301:
link.add_problem(str(response.status) + ": " + response.reason)
# determin depth
redirectdepth = 0
for p in link.parents:
redirectdepth = max(redirectdepth,p.redirectdepth)
link.redirectdepth = redirectdepth + 1
# check depth
if link.redirectdepth >= config.REDIRECT_DEPTH:
link.add_problem("too many redirects (%d)" % link.redirectdepth)
return None
# find url that is redirected to
location = urlparse.urljoin(link.url,response.getheader("Location",""))
if location == link.url:
link.add_problem("redirect same as source: %s" % location)
return None
# add child
link.add_child(location)
return None
# FIXME: add check for redirect loop detection
elif response.status != 200:
# handle error responses
link.add_problem(str(response.status) + ": " + response.reason)
return None
elif link.mimetype in acceptedtypes:
# return succesful responses
# TODO: support gzipped content
# TODO: add checking for size
return response.read()
except httplib.BadStatusLine, e:
link.add_problem("error reading HTTP response: "+str(e))
return None
except socket.error, (errnr,errmsg):
link.add_problem("error reading HTTP response: "+errmsg)
return None
finally:
# close the connection before returning
if conn is not None:
conn.close()
|