From 6767777944d34ff92548a4a873256ef3cd768982 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Fri, 23 Apr 2021 20:17:06 +1000 Subject: [PATCH] Catch requests errors in site downloaders --- bdfr/site_downloaders/base_downloader.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bdfr/site_downloaders/base_downloader.py b/bdfr/site_downloaders/base_downloader.py index ac45dc3..10787b8 100644 --- a/bdfr/site_downloaders/base_downloader.py +++ b/bdfr/site_downloaders/base_downloader.py @@ -8,7 +8,7 @@ from typing import Optional import requests from praw.models import Submission -from bdfr.exceptions import ResourceNotFound +from bdfr.exceptions import ResourceNotFound, SiteDownloaderError from bdfr.resource import Resource from bdfr.site_authenticator import SiteAuthenticator @@ -27,7 +27,11 @@ class BaseDownloader(ABC): @staticmethod def retrieve_url(url: str, cookies: dict = None, headers: dict = None) -> requests.Response: - res = requests.get(url, cookies=cookies, headers=headers) + try: + res = requests.get(url, cookies=cookies, headers=headers) + except requests.exceptions.RequestException as e: + logger.exception(e) + raise SiteDownloaderError(f'Failed to get page {url}') if res.status_code != 200: raise ResourceNotFound(f'Server responded with {res.status_code} to {url}') return res