1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
|
#!/usr/bin/env python
# webcheck.py - main module of webcheck doing command-line checking
#
# Copyright (C) 1998, 1999 Albert Hopkins (marduk)
# Copyright (C) 2002 Mike W. Meyer
# Copyright (C) 2005 Arthur de Jong
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
import config
import crawler
import plugins
import debugio
import sys
import time
import os
debugio.loglevel=debugio.INFO
def print_version():
"""print version information"""
print \
"webcheck "+config.VERSION+"\n" \
"Written by Albert Hopkins (marduk), Mike W. Meyer and Arthur de Jong.\n" \
"\n" \
"Copyright (C) 1998, 1999, 2002, 2005 Albert Hopkins (marduk), Mike W. Meyer\n" \
"and Arthur de Jong.\n" \
"This is free software; see the source for copying conditions. There is NO\n" \
"warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
def print_usage():
"""print short usage information"""
print >>sys.stderr, \
"Usage: webcheck [OPTION]... URL..."
def print_tryhelp():
"""print friendly pointer to more information"""
print >>sys.stderr, \
"Try `webcheck --help' for more information."
def print_help():
"""print option list"""
print \
"Usage: webcheck [OPTION]... URL...\n" \
"Generate a report for the given URLs\n" \
"\n" \
" -x PATTERN mark URLs matching PATTERN as external\n" \
" -y PATTERN do not check URLs matching PATTERN\n" \
" -b base URLs only: consider any URL not starting with the base\n" \
" URL to be external\n" \
" -a do not check external URLs\n" \
" -q, --quiet, --silent\n" \
" do not print out progress as webcheck traverses a site\n" \
" -d, --debug set loglevel to LEVEL, for programmer-level debugging\n" \
" -o DIRECTORY the directory in which webcheck will generate the reports\n" \
" -f, --force overwrite files without asking\n" \
" -r N the amount of redirects webcheck should follow when following\n" \
" a link, 0 implies follow all redirects.\n" \
" -w, --wait=SECONDS\n" \
" wait SECONDS between retrievals\n" \
" -V, --version output version information and exit\n" \
" -h, --help display this help and exit"
def parse_args(site):
"""parse command-line arguments"""
import getopt
try:
optlist, args = getopt.gnu_getopt(sys.argv[1:],
"x:y:l:baqdo:fr:w:Vh",
["quiet","silent","debug","force","wait=","version","help"])
except getopt.error, reason:
print >>sys.stderr,"webcheck: %s" % reason;
print_tryhelp()
sys.exit(1)
for flag,arg in optlist:
if flag=='-x':
site.add_external_re(arg)
elif flag=='-y':
site.add_yanked_re(arg)
elif flag=='-b':
config.BASE_URLS_ONLY = True
elif flag=='-a':
config.AVOID_EXTERNAL_LINKS = True
elif flag in ("-q","--quiet","--silent"):
debugio.loglevel = debugio.ERROR
elif flag in("-d","--debug"):
debugio.loglevel = debugio.DEBUG
elif flag=='-o':
config.OUTPUT_DIR = arg
elif flag in ("-f","--force"):
config.OVERWRITE_FILES = True
elif flag=='-r':
config.REDIRECT_DEPTH = int(arg)
elif flag in ("-w","--wait"):
config.WAIT_BETWEEN_REQUESTS = int(arg)
elif flag in ("-V","--version"):
print_version()
sys.exit(0)
elif flag in ("-h","--help"):
print_help()
sys.exit(0)
if len(args)==0:
print_usage()
print_tryhelp()
sys.exit(1)
for arg in args:
# if it does not look like a url it is probably a local file
import urlparse
import urllib
if urlparse.urlsplit(arg)[0] == '':
arg = 'file://' + urllib.pathname2url(os.path.abspath(arg))
site.add_internal(arg)
def install_file(source, text=False):
"""Install the given file in the output directory.
If the text flag is set to true it is assumed the file is text,
translating line endings."""
import shutil
import urlparse
# figure out mode to open the file with
mode='r'
if text:
mode+='U'
# check with what kind of argument we are called
scheme = urlparse.urlsplit(source)[0]
if scheme == 'file':
# this is a file:/// url, translate to normal path and open
source = urllib.url2pathname(urlparse.urlsplit(source)[2])
elif scheme == '' and os.path.isabs(source):
# this is an absolute path, just open it as is
pass
elif scheme == '':
# this is a relavite path, try to fetch it from the python path
for d in sys.path:
tst = os.path.join(d,source)
if os.path.isfile(tst):
source = tst
break
# TODO: support more schemes here
# figure out the destination name
target = os.path.join(config.OUTPUT_DIR, os.path.basename(source))
# test if source and target are the same
source = os.path.realpath(source)
if source == os.path.realpath(target):
debugio.error('attempt to overwrite %(fname)s with itself' % {'fname': source})
return
# open the input file
sfp = None
try:
sfp = open(source, mode)
except IOError, (errno, strerror):
debugio.error('error opening %(fname)s: %(strerror)s' %
{ 'fname': source,
'strerror': strerror })
sys.exit(1)
# create file in output directory (with overwrite question)
tfp=plugins.open_file(target);
# copy contents
shutil.copyfileobj(sfp,tfp)
# close files
tfp.close()
sfp.close()
def main():
site = crawler.Site()
# parse command-line arguments
parse_args(site)
# crawl through the website
debugio.info('checking site....')
try:
site.crawl() # this will take a while
except KeyboardInterrupt:
sys.stderr.write("Interrupted\n")
sys.exit(1)
debugio.info('done.')
# now we can write out the files
# start with the frame-description page
debugio.info('generating reports...')
# for every plugin, generate a page
plugins.generate(site, config.PLUGINS)
# put extra files in the output directory
install_file('webcheck.css',True)
debugio.info('done.')
if __name__ == '__main__':
main()
|