Fixed misc bugs

bdfr/site_downloaders/base_downloader.py

@@ -27,11 +27,23 @@ class BaseDownloader(ABC):
 
     @staticmethod
     def retrieve_url(url: str, cookies: dict = None, headers: dict = None) -> requests.Response:
-        try:
-            res = requests.get(url, cookies=cookies, headers=headers)
-        except requests.exceptions.RequestException as e:
-            logger.exception(e)
-            raise SiteDownloaderError(f"Failed to get page {url}")
-        if res.status_code != 200:
-            raise ResourceNotFound(f"Server responded with {res.status_code} to {url}")
-        return res
+        max_retries = 3
+        for attempt in range(1, max_retries + 1):
+            try:
+                res = requests.get(url, cookies=cookies, headers=headers, timeout=10)
+                if res.status_code != 200:
+                    logger.error(f"Attempt {attempt}: Server responded with {res.status_code} to {url}")
+                    if attempt == max_retries:
+                        raise ResourceNotFound(f"Server responded with {res.status_code} to {url}")
+                else:
+                    return res
+            except requests.exceptions.SSLError as ssl_err:
+                logger.error(f"Attempt {attempt}: SSL error for {url}: {ssl_err}")
+                if attempt == max_retries:
+                    raise SiteDownloaderError(f"SSL error after {max_retries} attempts for {url}: {ssl_err}")
+            except requests.exceptions.RequestException as e:
+                logger.error(f"Attempt {attempt}: Request error for {url}: {e}")
+                if attempt == max_retries:
+                    raise SiteDownloaderError(f"Failed to get page {url} after {max_retries} attempts: {e}")
+        # Should not reach here
+        raise SiteDownloaderError(f"Failed to get page {url} after {max_retries} attempts")