From e57b27612f2df0ade2d8f4dba62943712d6b74b4 Mon Sep 17 00:00:00 2001 From: parazyd Date: Sun, 23 Jul 2017 17:58:47 +0200 Subject: [PATCH] fix multiprocessing issue with download() the process should not forcefully exit with die() since it causes a deadlock for multiprocessing.Pool this commit also reimplements handling of requests.exceptions.ReadTimeout --- lib/net.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/net.py b/lib/net.py index 085b1ac..3f16160 100755 --- a/lib/net.py +++ b/lib/net.py @@ -21,16 +21,14 @@ def download(uris): try: r = requests.get(url, stream=True, timeout=20) - # TODO: investigate also requests.exceptions.ReadTimeout - except requests.exceptions.ConnectionError as e: + except (requests.exceptions.ConnectionError, + requests.exceptions.ReadTimeout) as e: warn('Caught exception: "%s". Retrying...' % e) return download(uris) - if r.status_code == 404: - warn('failed: 404 not found!') + if r.status_code != 200: + warn('%s failed: %d' % (url, r.status_code)) return - elif r.status_code != 200: - die('failed: %d' % r.status_code) makedirs(dirname(path), exist_ok=True) f = open(path, 'wb')