path: root/myUrlLib.py

# Copyright (C) 1998,1999  marduk <marduk@python.net>
# Copyright (C) 2002 Mike Meyer <mwm@mired.org>

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

"""Generic library for handling urls and links"""

config = None
robot_parsers={}
SECS_PER_DAY=60*60*24
compiled_ex = []
compiled_yanked = []
linkmodules={}

from urllib import *
from types import *
import htmllib
import httplib
import robotparser
import string
# The following is to help sgmllib parse DOS/Windows text files
string.whitespace = string.whitespace + '\012\015'
import time
import re
import stat
import htmlparse
import debugio
import sys
import socket


def get_robots(location):
    global robot_parsers
    debugio.write('\tGetting robots.txt for %s' % location)
    rp=robotparser.RobotFileParser(config.PROXIES)
    try:
        rp.set_url('http://' + location + '/robots.txt')
        rp.read()
        robot_parsers[location]=rp
    except TypeError:
        pass

def can_fetch(location, url):
    """Return true if url is allowed at location, else return 0"""
    if robot_parsers.has_key(location):
        return robot_parsers[location].can_fetch('Webcheck',url)
    return 1

############################################################################
class Link:
    """ my class of url's which includes parents, HTTP status number, and
        a list of URL's in that link urls.
    """

    linkList = {}
    badLinks = []
    notChecked = []
    images = {}
    baseurl = ""
    base=""

    # This is a static variable to indicate if the config.EXCLUDED urls have been
    # compiled as regular expressions.
    re_compiled = 0

    def __init__(self,url,parent):
        self.init()

        debugio.write('\tparent = ' + str(parent),2)
        from urlparse import urlparse

        parsed = urlparse(url)
        self.scheme = parsed[0]
        location = parsed[1]

        if parent not in self.parents:
            if parent: self.parents.append(parent)
            
        self.URL = url
        Link.linkList[self.URL]=self

        modname = self.scheme + 'link'
        if linkmodules.has_key(modname): linkmodule = linkmodules[modname]
        else: 
            try:
                linkmodule = linkmodules[modname] = __import__('schemes.'+modname, globals(),locals(),[modname])
            except ImportError:
                self.status="Not Checked"
                self.external=1
                self.URL=url
                Link.notChecked.append(self.URL)
                Link.linkList[self.URL]=self
                debugio.write('\tNot checked: URL scheme ' + self.scheme + ' ignored.')
                return
        
        if (parent is None):
            Link.baseurl=self.URL
            if hasattr(self.URL, 'rfind'):
                Link.base=self.URL[:self.URL.rfind('/')+1]
            else:
                Link.base=self.URL[:string.rfind(self.URL,'/')+1]
            if Link.base[-2:] == '//': Link.base = self.URL
            debugio.write('\tbase: %s' % Link.base)
            if self.scheme == 'http':
                base_location = parsed[1]
                if base_location not in config.HOSTS:
                    config.HOSTS.append(base_location)
                if not robot_parsers.has_key(location):
                    try:
                        get_robots(location)
                    except IOError:
                        pass

        # see if robots.txt will let us in
        if self.scheme == 'http':
            if not can_fetch(location, url):
                debugio.write('\tRobot Restriced')
                self.status = 'Not Checked'
                self.message = 'Robot Restricted'
                Link.notChecked.append(url)
                return

        try:
            linkmodule.init(self, url, parent)
            if (self.URL not in Link.badLinks) and (self.type == 'text/html'):
                page = linkmodule.get_document(self.URL)
                self._handleHTML(self.URL, page)
        except IOError, data:
            self.set_bad_link(url,str(data.errno) + ': ' + str(data.strerror))
            return
        except socket.error, data:
            if type(data) is StringType:
                self.set_bad_link(url, data)
            elif type(data) is TupleType:
                errno, string = data
                self.set_bad_link(url,str(errno) + ': ' + string)
            else:
                self.set_bad_link(url,str(data))
        except KeyboardInterrupt:
            raise KeyboardInterrupt
        except:
            self.set_bad_link(url,"Error: Malformed URL?")
            debugio.write("\t%s: %s" % (sys.exc_type, sys.exc_value),3)
            return
        
    def explore_children(self):
        for child in self.children:
            if not Link.linkList.has_key(child):
                if config.WAIT_BETWEEN_REQUESTS > 0:
                    debugio.write('sleeping %s seconds' %  config.WAIT_BETWEEN_REQUESTS)
                    time.sleep(config.WAIT_BETWEEN_REQUESTS)
                debugio.write("adding url: %s" % child)
                if is_yanked(child):
                    Link.linkList[child]=ExternalLink(child,self.URL,1)
                elif is_external(child) or is_excluded(child):
                    Link.linkList[child]=ExternalLink(child,self.URL)
                else:
                    Link.linkList[child]=Link(child,self.URL)
            elif self.URL not in Link.linkList[child].parents:
                Link.linkList[child].parents.append(self.URL)
        return # __init__

    def init(self):
        """ initialize some variables """
        self.age = None
        self.scheme = None
        self.headers = None
        self.parents= []
        self.children = []
        self.status = None
        self.title = None
        self.external = 0
        self.html = 0
        self.size = 0
        self.totalSize = 0
        self.author = None

    def __repr__(self):
        return self.URL

    def set_bad_link(self,url,status):
        """ flags the link as bad """
        debugio.write('\t' + str(status))
        self.status = str(status)
        self.URL=url
        Link.linkList[self.URL]=self
        Link.badLinks.append(self.URL)

    def _handleHTML(self,url,htmlfile):
        """examines and html file and updates the Link object"""
        # get anchorlist
        (anchorlist, imagelist, title, author) = htmlparse.pageLinks(url,htmlfile)

        debugio.write('\ttitle: %s' % str(title))
        for child in anchorlist:
            if child not in self.children:
                self.children.append(child)
                    
        self.totalSize = self.size
        self.title = title
        self.author = author
        self.html = 1
        # get image list
        for image in imagelist:
            if image not in Link.images.keys():
                debugio.write('\tadding image: %s' % image)
                Link.images[image] = Image(image, self.URL)
            self.totalSize = self.totalSize + int(Link.images[image].size)
        if not self.external: self.explore_children()
        return



class ExternalLink(Link):
    """ this class is just like Link, but it does not explore it's children """
    
    def __init__(self,url,parent,yanked=0):

        if config.AVOID_EXTERNAL_LINKS or yanked:
            self.init()
            self.status="Not Checked"
            self.external=1
            debugio.write('\tNot checked')
            if yanked: debugio.write('\tYanked')
            if parent not in self.parents:
                if parent: self.parents.append(parent)
            Link.notChecked.append(url)
            return
        Link.__init__(self,url,parent)
        self.external=1


    def _handleHTML(self,url,htmlfile):
        # ignore links and images, but use the title
        self.title = htmlparse.pageLinks(url,htmlfile)[2]
        debugio.write('\ttitle: %s' % str(self.title))
        self.children=[]

class Image(Link):
    """ This class is just like link, but different :-)"""
    def __init__(self, url, parent):
        #self.init()
        Link.__init__(self, url, parent)
        #self.age = getAge(self)

    def _handleHTML(self,url,htmlfile):
        """Don't handle HTML, this is an image"""
        self.set_bad_link(url,"HTML file used in IMG tag?")
        return

def is_external(url):
    """ returns true if url is an external link """
    from urlparse import urlparse
    parsed = urlparse(url)
    scheme = parsed[0]
    location = parsed[1]
    if (location not in config.HOSTS) and (scheme in ['http','ftp']):
        return 1
    if config.BASE_URLS_ONLY and (Link.base!=url[:len(Link.base)]):
        return 1
    return 0

def compile_re():
    """Compile EXCLUDED URLSs and set flag"""
    global compiled_ex
    for i in config.EXCLUDED_URLS:
        debugio.write('compiling %s' % i,3)
        compiled_ex.append(re.compile(i,re.IGNORECASE))
    for i in config.YANKED_URLS:
        debugio.write('compiling %s' % i,3)
        compiled_yanked.append(re.compile(i,re.IGNORECASE))
    Link.re_compiled = 1

def is_excluded(url):
    """ Returns true if url is part of the EXCLUDED_URLS list """
    if not Link.re_compiled:
        compile_re()
    for x in compiled_ex:
        if x.search(url) is not None:
            return 1
    return 0

def is_yanked(url):
    """ Returns true if url is part of YANKED_URLS list"""
    if not Link.re_compiled:
        compile_re()
    for x in compiled_yanked:
        if x.search(url) is not None:
            return 1
    return 0