From d26d8299a054be7be491c7210714291c05083e60 Mon Sep 17 00:00:00 2001 From: Arthur de Jong Date: Wed, 29 Aug 2012 15:34:23 +0000 Subject: pass a string to RobotFileParser because of problems with unicode git-svn-id: http://arthurdejong.org/svn/webcheck/webcheck@471 86f53f14-5ff3-0310-afe5-9b438ce3f40c --- webcheck/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webcheck/crawler.py b/webcheck/crawler.py index 3f0c3b8..0099399 100644 --- a/webcheck/crawler.py +++ b/webcheck/crawler.py @@ -285,7 +285,7 @@ class Crawler(object): # choose a link from the tocheck list link = tocheck.pop() link.is_internal = self._is_internal(link.url) - link.yanked = self._is_yanked(link.url) + link.yanked = self._is_yanked(str(link.url)) # see if there are any more links to check if not tocheck: tocheck = self.get_links_to_crawl(session) -- cgit v1.2.3