path: root/crawler.py

# crawler.py - definition of Link class for storing the crawled site
#
# Copyright (C) 1998, 1999 Albert Hopkins (marduk)
# Copyright (C) 2002 Mike W. Meyer
# Copyright (C) 2005, 2006, 2007, 2008, 2011 Arthur de Jong
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
#
# The files produced as output from the software do not automatically fall
# under the copyright of the software, unless explicitly stated otherwise.

"""General module to do site-checking. This module contains the Site class
containing the state for the crawled site and some functions to access and
manipulate the crawling of the website. This module also contains the Link
class that holds all the link related properties."""

import atexit
import cookielib
import datetime
import httplib
import os
import re
import robotparser
import socket
import time
import urllib
import urllib2
import urlparse

import config
import db
import debugio
import parsers


class RedirectError(urllib2.HTTPError):
    def __init__(self, url, code, msg, hdrs, fp, newurl):
        self.newurl = newurl
        urllib2.HTTPError.__init__(self, url, code, msg, hdrs, fp)


class NoRedirectHandler(urllib2.HTTPRedirectHandler):

    def redirect_request(self, req, fp, code, msg, headers, newurl):
        raise RedirectError(req.get_full_url(), code, msg, headers, fp, newurl)


def setup_urllib2():
    """Configure the urllib2 module to store cookies in the output
    directory."""
    filename = os.path.join(config.OUTPUT_DIR, 'cookies.lwp')
    # set up our cookie jar
    cookiejar = cookielib.LWPCookieJar(filename)
    try:
        cookiejar.load(ignore_discard=False, ignore_expires=False)
    except IOError:
        pass
    atexit.register(cookiejar.save, ignore_discard=False, ignore_expires=False)
    # set up our custom opener that sets a meaningful user agent
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar),
                                  NoRedirectHandler())
    opener.addheaders = [
      ('User-agent', 'webcheck %s' % config.VERSION),
      ]
    if config.BYPASSHTTPCACHE:
        opener.addheaders.append(('Cache-control', 'no-cache'))
        opener.addheaders.append(('Pragma', 'no-cache'))
    urllib2.install_opener(opener)


# pattern for matching spaces
_spacepattern = re.compile(' ')

# pattern to match anchor part of a url
_anchorpattern = re.compile('#([^#]+)$')


# TODO: rename Site to Crawler
class Site(object):
    """Class to represent gathered data of a site.

    The available properties of this class are:

      bases      - a list of base link object
   """

    def __init__(self):
        """Creates an instance of the Site class and initializes the
        state of the site."""
        # list of internal urls
        self._internal_urls = set()
        # list of regexps considered internal
        self._internal_res = {}
        # list of regexps considered external
        self._external_res = {}
        # list of regexps matching links that should not be checked
        self._yanked_res = {}
        # map of scheme+netloc to robot handleds
        self._robotparsers = {}
        # list of base urls (these are the internal urls to start from)
        self.bases = []

    def add_internal(self, url):
        """Add the given url and consider all urls below it to be internal.
        These links are all marked for checking with the crawl() function."""
        url = db.Link.clean_url(url)
        if url not in self._internal_urls:
            self._internal_urls.add(url)

    def add_internal_re(self, exp):
        """Adds the gived regular expression as a pattern to match internal
        urls."""
        self._internal_res[exp] = re.compile(exp, re.IGNORECASE)

    def add_external_re(self, exp):
        """Adds the gived regular expression as a pattern to match external
        urls."""
        self._external_res[exp] = re.compile(exp, re.IGNORECASE)

    def add_yanked_re(self, exp):
        """Adds the gived regular expression as a pattern to match urls that
        will not be checked at all."""
        self._yanked_res[exp] = re.compile(exp, re.IGNORECASE)

    def _is_internal(self, url):
        """Check whether the specified url is external or internal.
        This uses the urls marked with add_internal() and the regular
        expressions passed with add_external_re()."""
        # check if it is internal through the regexps
        for regexp in self._internal_res.values():
            if regexp.search(url) is not None:
                return True
        res = False
        # check that the url starts with an internal url
        if config.BASE_URLS_ONLY:
            # the url must start with one of the _internal_urls
            for i in self._internal_urls:
                res |= (i == url[:len(i)])
        else:
            # the netloc must match a netloc of an _internal_url
            netloc = urlparse.urlsplit(url)[1]
            for i in self._internal_urls:
                res |= (urlparse.urlsplit(i)[1] == netloc)
        # if it is not internal now, it never will be
        if not res:
            return False
        # check if it is external through the regexps
        for x in self._external_res.values():
            # if the url matches it is external and we can stop
            if x.search(url):
                return False
        return True

    def _get_robotparser(self, scheme, netloc):
        """Return the proper robots parser for the given url or None if one
        cannot be constructed. Robot parsers are cached per scheme and
        netloc."""
        # only some schemes have a meaningful robots.txt file
        if scheme != 'http' and scheme != 'https':
            debugio.debug('crawler._get_robotparser() '
                          'called with unsupported scheme (%s)' % scheme)
            return None
        # split out the key part of the url
        location = urlparse.urlunsplit((scheme, netloc, '', '', ''))
        # try to create a new robotparser if we don't already have one
        if location not in self._robotparsers:
            debugio.info('  getting robots.txt for %s' % location)
            self._robotparsers[location] = None
            try:
                rp = robotparser.RobotFileParser()
                rp.set_url(urlparse.urlunsplit(
                  (scheme, netloc, '/robots.txt', '', '')))
                rp.read()
                self._robotparsers[location] = rp
            except (TypeError, IOError, httplib.HTTPException):
                # ignore any problems setting up robot parser
                pass
        return self._robotparsers[location]

    def _is_yanked(self, url):
        """Check whether the specified url should not be checked at all.
        This uses the regualr expressions passed with add_yanked_re() and the
        robots information present."""
        # check if it is yanked through the regexps
        for regexp in self._yanked_res.values():
            # if the url matches it is yanked and we can stop
            if regexp.search(url):
                return 'yanked'
        # check if we should avoid external links
        is_internal = self._is_internal(url)
        if not is_internal and config.AVOID_EXTERNAL_LINKS:
            return 'external avoided'
        # check if we should use robot parsers
        if not config.USE_ROBOTS:
            return None
        (scheme, netloc) = urlparse.urlsplit(url)[0:2]
        # skip schemes not having robot.txt files
        if scheme not in ('http', 'https'):
            return None
        # skip robot checks for external urls
        # TODO: make this configurable
        if not is_internal:
            return None
        # check robots for remaining links
        rp = self._get_robotparser(scheme, netloc)
        if rp and not rp.can_fetch('webcheck', url):
            return 'robot restriced'
        # fall back to allowing the url
        return None

    def get_link(self, session, url):
        # try to find the URL
        url = db.Link.clean_url(url)
        link = session.query(db.Link).filter_by(url=url).first()
        if not link:
            link = db.Link(url=url)
            session.add(link)
        return link

    def get_links_to_crawl(self, session):
        links = session.query(db.Link).filter(db.Link.fetched == None)
        return links.filter(db.Link.yanked == None)

    def crawl(self):
        """Crawl the website based on the urls specified with
        add_internal(). If the serialization file pointer
        is specified the crawler writes out updated links to
        the file while crawling the site."""
        # get a database session
        session = db.Session()
        # remove all links
        if not config.CONTINUE:
            session.query(db.LinkProblem).delete()
            session.commit()
            session.query(db.PageProblem).delete()
            session.commit()
            session.execute(db.children.delete())
            session.commit()
            session.execute(db.embedded.delete())
            session.commit()
            session.query(db.Link).delete()
            session.commit()
        # add all internal urls to the database
        for url in self._internal_urls:
            url = db.Link.clean_url(url)
            self.get_link(session, url)
        # add some URLs from the database that haven't been fetched
        tocheck = self.get_links_to_crawl(session)
        remaining = tocheck.count()
        tocheck = tocheck[:100]
        remaining -= len(tocheck)
        # repeat until we have nothing more to check
        while tocheck:
            # choose a link from the tocheck list
            link = tocheck.pop()
            link.is_internal = self._is_internal(link.url)
            link.yanked = self._is_yanked(link.url)
            # see if there are any more links to check
            if not tocheck:
                tocheck = self.get_links_to_crawl(session)
                remaining = tocheck.count()
                tocheck = tocheck[:100]
                remaining -= len(tocheck)
            # skip link it there is nothing to check
            if link.yanked or link.fetched:
                continue
            # fetch the link's contents
            response = self.fetch(link)
            if response:
                self.parse(link, response)
            # flush database changes
            session.commit()
            # sleep between requests if configured
            if config.WAIT_BETWEEN_REQUESTS > 0:
                debugio.debug('crawler.crawl(): sleeping %s seconds' %
                              config.WAIT_BETWEEN_REQUESTS)
                time.sleep(config.WAIT_BETWEEN_REQUESTS)
            debugio.debug('crawler.crawl(): items left to check: %d' %
                          (remaining + len(tocheck)))
        session.commit()

    def fetch(self, link):
        """Attempt to fetch the url (if not yanked) and fill in link
        attributes (based on is_internal)."""
        debugio.info('  %s' % link.url)
        # mark the link as fetched to avoid loops
        link.fetched = datetime.datetime.now()
        # see if we can import the proper module for this scheme
        try:
            # FIXME: if an URI has a username:passwd add the uri, username and password to the HTTPPasswordMgr
            request = urllib2.Request(link.url)
            parent = link.parents.first()
            if parent:
                request.add_header('Referer', parent.url)
            response = urllib2.urlopen(request)
            link.mimetype = response.info().gettype()
            link.set_encoding(response.headers.getparam('charset'))
            # FIXME: get result code and other stuff
            link.status = str(response.code)
            # link.size = int(response.getheader('Content-length'))
            # link.mtime = time.mktime(response.msg.getdate('Last-Modified'))
            # if response.status == 301: link.add_linkproblem(str(response.status)+': '+response.reason)
            # elif response.status != 200: link.add_linkproblem(str(response.status)+': '+response.reason)
            # TODO: add checking for size
            return response
        except RedirectError, e:
            link.status = str(e.code)
            debugio.info('    ' + str(e))
            if e.code == 301:
                link.add_linkproblem(str(e))
            link.add_redirect(e.newurl)
            return
        except urllib2.HTTPError, e:
            link.status = str(e.code)
            debugio.info('    ' + str(e))
            link.add_linkproblem(str(e))
            return
        except urllib2.URLError, e:
            debugio.info('    ' + str(e))
            link.add_linkproblem(str(e))
            return
        except KeyboardInterrupt:
            # handle this in a higher-level exception handler
            raise
        except Exception, e:
            # handle all other exceptions
            debugio.warn('unknown exception caught: ' + str(e))
            link.add_linkproblem('error reading HTTP response: %s' % str(e))
            import traceback
            traceback.print_exc()
            return

    def parse(self, link, response):
        """Parse the fetched response."""
        # find a parser for the content-type
        parsermodule = parsers.get_parsermodule(link.mimetype)
        if parsermodule is None:
            debugio.debug('crawler.Link.fetch(): unsupported content-type: %s' % link.mimetype)
            return
        # skip parsing of content if we were returned nothing
        content = response.read()
        if content is None:
            return
        # parse the content
        debugio.debug('crawler.Link.fetch(): parsing using %s' % parsermodule.__name__)
        try:
            parsermodule.parse(content, link)
        except Exception, e:
            import traceback
            traceback.print_exc()
            debugio.warn('problem parsing page: ' + str(e))
            link.add_pageproblem('problem parsing page: ' + str(e))

    def postprocess(self):
        """Do some basic post processing of the collected data, including
        depth calculation of every link."""
        # get a database session
        session = db.Session()
        # build the list of urls that were set up with add_internal() that
        # do not have a parent (they form the base for the site)
        for url in self._internal_urls:
            link = self.get_link(session, url).follow_link()
            if not link:
                debugio.warn('base link %s redirects to nowhere' % url)
                continue
            # add the link to bases
            debugio.debug('crawler.postprocess(): adding %s to bases' % link.url)
            self.bases.append(link)
        # if we got no bases, just use the first internal one
        if not self.bases:
            link = session.query(db.Link).filter(db.Link.is_internal == True).first()
            debugio.debug('crawler.postprocess(): fallback to adding %s to bases' % link.url)
            self.bases.append(link)
        # do a breadth first traversal of the website to determine depth
        session.query(db.Link).update(dict(depth=None), synchronize_session=False)
        session.commit()
        depth = 0
        count = len(self.bases)
        for link in self.bases:
            link.depth = 0
        session.commit()
        debugio.debug('crawler.postprocess(): %d links at depth 0' % count)
        while count > 0:
            # update the depth of all links without a depth that have a
            # parent with the previous depth
            qry = session.query(db.Link).filter(db.Link.depth == None)
            qry = qry.filter(db.Link.linked_from.any(db.Link.depth == depth))
            count = qry.update(dict(depth=depth + 1), synchronize_session=False)
            session.commit()
            depth += 1
            debugio.debug('crawler.postprocess(): %d links at depth %d' % (count, depth))
            # TODO: also handle embeds
        # see if any of the plugins want to do postprocessing
        for p in config.PLUGINS:
            # import the plugin
            plugin = __import__('plugins.' + p, globals(), locals(), [p])
            if hasattr(plugin, 'postprocess'):
                debugio.info('  ' + p)
                plugin.postprocess(self)

    def generate(self):
        """Generate pages for plugins."""
        for p in config.PLUGINS:
            # import the plugin
            plugin = __import__('plugins.' + p, globals(), locals(), [p])
            if hasattr(plugin, 'generate'):
                debugio.info('  ' + p)
                plugin.generate(self)