From 621e9d699882b4ef4a6e72324ecc8dd1310d16d1 Mon Sep 17 00:00:00 2001
From: Arthur de Jong <arthur@arthurdejong.org>
Date: Thu, 2 Apr 2015 18:52:40 +0200
Subject: Ingore errors in crawling directories

---
 crawler.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/crawler.py b/crawler.py
index 32b0e5b..3f36bc8 100644
--- a/crawler.py
+++ b/crawler.py
@@ -72,11 +72,17 @@ def walk(top, path_excludes, dir_excludes):
             print('%r %r' % (top, name))
     # recurse into directories
     for name in dirs:
-        os.chdir(name)
-        for path, size, is_dir, meta in walk(os.path.join(top, name),
-                                             path_excludes, dir_excludes):
-            yield path, size, is_dir, meta
-        os.chdir('..')
+        try:
+            os.chdir(name)
+        except (UnicodeDecodeError, OSError):
+            import traceback
+            print(traceback.format_exc())
+            print('%r %r' % (top, name))
+        else:
+            for path, size, is_dir, meta in walk(os.path.join(top, name),
+                                                 path_excludes, dir_excludes):
+                yield path, size, is_dir, meta
+            os.chdir('..')
 
 
 def crawl(paths, excludes=()):
-- 
cgit v1.2.3