1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
|
# ftp.py - handle urls with a ftp scheme
#
# Copyright (C) 1998, 1999 Albert Hopkins (marduk)
# Copyright (C) 2002 Mike W. Meyer
# Copyright (C) 2005 Arthur de Jong
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
#
# The files produced as output from the software do not automatically fall
# under the copyright of the software, unless explicitly stated otherwise.
"""This module defines the functions needed for creating Link objects for urls
using the ftp scheme."""
import config
import urllib
import mimetypes
import ftplib
import urlparse
import string
import debugio
# FIXME: honor ftp proxy settings
# TODO: figure out a nicer way to close the connections with something like:
#try:
# debugio.debug("FTP: "+ftp.quit())
#except:
# debugio.debug("FTP: "+ftp.close())
# TODO: handle some exceptions earlier or use different commands
# a map of netlocs to ftp connections
_ftpconnections = {}
def _getconnection(netloc):
"""Return a FTP connection object to the specified server."""
# NOTE: this method is not thread safe
if _ftpconnections.has_key(netloc):
return _ftpconnections[netloc]
# split url into useful parts
(userpass, host) = urllib.splituser(netloc)
if userpass is not None:
(user, passwd) = urllib.splitpasswd(userpass)
else:
(user, passwd) = ('anonymous', '')
(host, port) = urllib.splitnport(host,ftplib.FTP_PORT)
# initialize a new connection
ftp = ftplib.FTP()
debugio.debug('schemes.ftp._getconnection(): CONNECT: '+ftp.connect(host, port))
debugio.debug('schemes.ftp._getconnection(): LOGIN: '+ftp.login(user, passwd))
_ftpconnections[netloc] = ftp
return ftp
def _cwd(ftp, path):
"""Go down the path on the ftp server returning the part that cannot be
changed into."""
# split the path into directories
dirs = path.split('/')
try:
# decend down the tree
while len(dirs) > 0:
d = dirs[0]
if d != '':
debugio.debug('schemes.ftp._cwd(): CWD '+d+': '+ftp.cwd(d))
dirs.pop(0)
return None
except ftplib.error_perm, e:
debugio.debug('schemes.ftp._cwd(): CWD '+d+': '+str(e))
return string.join(dirs,'/')
def _fetch_directory(link, ftp, acceptedtypes):
"""Handle the ftp directory."""
# check that the url ends with a slash
if link.path[-1:] != '/':
debugio.debug('schemes.ftp._fetch_directory(): directory referenced without trailing slash')
link.redirect(urlparse.urljoin(link.url,link.path+'/'))
return
# retreive the contents of the directory
# FIXME: this raises an exception for empty directories, probably replace with own command
contents = ftp.nlst()
# check contents of directory for some common files
for f in config.FTP_INDEXES:
if f in contents:
debugio.debug('schemes.ftp._fetch_directory(): pick up %s from directory' % f)
# the the directory contains an index.html, use that
return _fetch_file(link, ftp, f, acceptedtypes)
# just add files in directory as children
debugio.debug('schemes.ftp._fetch_directory(): add files as children of this page')
link.ispage = True
debugio.debug('schemes.ftp._fetch_directory(): TYPE A: '+ftp.voidcmd('TYPE A'))
# FIXME: this raises an exception for empty directories
for f in contents:
link.add_child(urlparse.urljoin(link.url,urllib.quote(f)))
def _fetch_file(link, ftp, path, acceptedtypes):
"""Try to download the file in path that should be in the current
directory of the ftp instance. The path can also point to a non-existant
file or directory."""
# figure out the size of the document
link.size = ftp.size(path)
debugio.debug('schemes.ftp.fetch(): size='+str(link.size))
# guess the mimetype of the document
if link.mimetype is None:
link.mimetype = mimetypes.guess_type(path)[0]
# try to fetch file
if link.mimetype in acceptedtypes:
debugio.debug('schemes.ftp.fetch(): TYPE I: '+ftp.voidcmd('TYPE I'))
(conn, size) = ftp.ntransfercmd('RETR ' + path)
if size:
content = conn.makefile().read(size)
else:
content = conn.makefile().read()
debugio.debug('schemes.ftp.fetch(): fetched, size=%d' % len(content))
return content
def fetch(link, acceptedtypes):
"""Fetch the specified link."""
# try to fetch the document
try:
ftp = _getconnection(link.netloc)
debugio.debug('schemes.ftp.fetch(): CWD /: '+ftp.cwd('/'))
# descend down the directory tree as far as we can go
path=urllib.unquote(link.path)
path=_cwd(ftp, path)
# check if we are dealing with an (exising) directory
if path is None:
return _fetch_directory(link, ftp, acceptedtypes)
else:
return _fetch_file(link, ftp, path, acceptedtypes)
except ftplib.all_errors, e:
debugio.debug('schemes.ftp.fetch(): CAUGHT '+str(e))
link.add_linkproblem(str(e))
|