1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
|
#!/usr/bin/env python
# webcheck.py - main module of webcheck doing command-line checking
#
# Copyright (C) 1998, 1999 Albert Hopkins (marduk) <marduk@python.net>
# Copyright (C) 2002 Mike W. Meyer <mwm@mired.org>
# Copyright (C) 2005 Arthur de Jong <arthur@tiefighter.et.tudelft.nl>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
import config
import crawler
import plugins
import debugio
import sys
import time
import os
debugio.loglevel=debugio.INFO
def print_version():
"""print version information"""
print \
"webcheck "+config.VERSION+"\n" \
"Written by Albert Hopkins (marduk), Mike W. Meyer and Arthur de Jong.\n" \
"\n" \
"Copyright (C) 1998, 1999, 2002, 2005 Albert Hopkins (marduk), Mike W. Meyer\n" \
"and Arthur de Jong.\n" \
"This is free software; see the source for copying conditions. There is NO\n" \
"warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
def print_usage():
"""print short usage information"""
print >>sys.stderr, \
"Usage: webcheck [OPTION]... URL..."
def print_tryhelp():
"""print friendly pointer to more information"""
print >>sys.stderr, \
"Try `webcheck --help' for more information."
def print_help():
"""print option list"""
print \
"Usage: webcheck [OPTION]... URL...\n" \
"Generate a report for the given URLs\n" \
"\n" \
" -x PATTERN mark URLs matching PATTERN as external\n" \
" -y PATTERN do not check URLs matching PATTERN\n" \
" -b base URLs only: consider any URL not starting with the base\n" \
" URL to be external\n" \
" -a do not check external URLs\n" \
" -q, --quiet, --silent\n" \
" do not print out progress as webcheck traverses a site\n" \
" -d, --debug set loglevel to LEVEL, for programmer-level debugging\n" \
" -o DIRECTORY the directory in which webcheck will generate the reports\n" \
" -f, --force overwrite files without asking\n" \
" -r N the amount of redirects webcheck should follow when following\n" \
" a link, 0 implies follow all redirects.\n" \
" -w, --wait=SECONDS\n" \
" wait SECONDS between retrievals\n" \
" -V, --version output version information and exit\n" \
" -h, --help display this help and exit"
def parse_args():
"""parse command-line arguments"""
import getopt
global site
try:
optlist, args = getopt.gnu_getopt(sys.argv[1:],
"x:y:l:baqdo:fr:w:Vh",
["quiet","silent","debug","force","wait=","version","help"])
except getopt.error, reason:
print >>sys.stderr,"webcheck: %s" % reason;
print_tryhelp()
sys.exit(1)
for flag,arg in optlist:
if flag=='-x':
site.add_external_re(arg)
elif flag=='-y':
site.add_yanked_re(arg)
elif flag=='-b':
config.BASE_URLS_ONLY = True
elif flag=='-a':
config.AVOID_EXTERNAL_LINKS = True
elif flag in ("-q","--quiet","--silent"):
debugio.loglevel = debugio.ERROR
elif flag in("-d","--debug"):
debugio.loglevel = debugio.DEBUG
elif flag=='-o':
config.OUTPUT_DIR = arg
elif flag in ("-f","--force"):
config.OVERWRITE_FILES = True
elif flag=='-r':
config.REDIRECT_DEPTH = int(arg)
elif flag in ("-w","--wait"):
config.WAIT_BETWEEN_REQUESTS = int(arg)
elif flag in ("-V","--version"):
print_version()
sys.exit(0)
elif flag in ("-h","--help"):
print_help()
sys.exit(0)
if len(args)==0:
print_usage()
print_tryhelp()
sys.exit(1)
for arg in args:
site.add_internal(arg)
def find_file(fname):
"""Search the python path for the file name and return full path of the
file."""
for dname in sys.path:
res = os.path.join(dname,fname)
if os.path.isfile(res):
return res
return None
def install_file(fname,text=False):
"""Install the given file in the output directory.
If the text flag is set to true it is assumed the file is text,
translating line endings."""
import shutil
# TODO: extend so that if
# - filename has no slashes in it: search python path
# - filename starts with a known scheme: use that
# - filename starts with slash: treat is as a file://///// url
# TODO: make it possible to reference the original location instead of copying the file
# FIXME: check that source and target are different before opening file for writing
source = find_file(fname)
target = os.path.basename(fname)
# open the input file, TODO: use the scheme stuff for doing this
mode='r'
if text:
mode+='U'
sfp=open(source,mode)
# create file in output directory (with overwrite question)
tfp=plugins.open_file(target);
# copy contents
shutil.copyfileobj(sfp,tfp)
# close files
tfp.close()
sfp.close()
def main():
global site
site = crawler.Site()
# parse command-line arguments
parse_args()
# crawl through the website
debugio.info('checking site....')
try:
site.crawl() # this will take a while
except KeyboardInterrupt:
sys.stderr.write("Interrupted\n")
sys.exit(1)
debugio.info('done.')
# now we can write out the files
# start with the frame-description page
debugio.info('generating reports...')
# for every plugin, generate a page
plugins.generate(site, config.PLUGINS)
# put extra files in the output directory
install_file('webcheck.css',True)
debugio.info('done.')
if __name__ == '__main__':
main()
|