From 3aa740e979ffd976508009a8e4c80934eeb3eaeb Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Sat, 10 Dec 2022 12:36:54 -0500
Subject: [PATCH] Add soft fail on 5xx Prawcore errors.

---
 bdfr/connector.py  | 44 +++++++++++++++++++++++++-------------------
 bdfr/downloader.py | 16 +++++++++++-----
 2 files changed, 36 insertions(+), 24 deletions(-)

diff --git a/bdfr/connector.py b/bdfr/connector.py
index ea970db..e5d74a2 100644
--- a/bdfr/connector.py
+++ b/bdfr/connector.py
@@ -13,6 +13,7 @@ from abc import ABCMeta, abstractmethod
 from datetime import datetime
 from enum import Enum, auto
 from pathlib import Path
+from time import sleep
 from typing import Callable, Iterator
 
 import appdirs
@@ -353,26 +354,31 @@ class RedditConnector(metaclass=ABCMeta):
             generators = []
             for user in self.args.user:
                 try:
-                    self.check_user_existence(user)
-                except errors.BulkDownloaderException as e:
-                    logger.error(e)
-                    continue
-                if self.args.submitted:
-                    logger.debug(f"Retrieving submitted posts of user {self.args.user}")
-                    generators.append(
-                        self.create_filtered_listing_generator(
-                            self.reddit_instance.redditor(user).submissions,
+                    try:
+                        self.check_user_existence(user)
+                    except errors.BulkDownloaderException as e:
+                        logger.error(e)
+                        continue
+                    if self.args.submitted:
+                        logger.debug(f"Retrieving submitted posts of user {user}")
+                        generators.append(
+                            self.create_filtered_listing_generator(
+                                self.reddit_instance.redditor(user).submissions,
+                            )
                         )
-                    )
-                if not self.authenticated and any((self.args.upvoted, self.args.saved)):
-                    logger.warning("Accessing user lists requires authentication")
-                else:
-                    if self.args.upvoted:
-                        logger.debug(f"Retrieving upvoted posts of user {self.args.user}")
-                        generators.append(self.reddit_instance.redditor(user).upvoted(limit=self.args.limit))
-                    if self.args.saved:
-                        logger.debug(f"Retrieving saved posts of user {self.args.user}")
-                        generators.append(self.reddit_instance.redditor(user).saved(limit=self.args.limit))
+                    if not self.authenticated and any((self.args.upvoted, self.args.saved)):
+                        logger.warning("Accessing user lists requires authentication")
+                    else:
+                        if self.args.upvoted:
+                            logger.debug(f"Retrieving upvoted posts of user {user}")
+                            generators.append(self.reddit_instance.redditor(user).upvoted(limit=self.args.limit))
+                        if self.args.saved:
+                            logger.debug(f"Retrieving saved posts of user {user}")
+                            generators.append(self.reddit_instance.redditor(user).saved(limit=self.args.limit))
+                except prawcore.PrawcoreException as e:
+                    logger.error(f"User {user} failed to be retrieved due to a PRAW exception: {e}")
+                    logger.debug("Waiting 60 seconds to continue")
+                    sleep(60)
             return generators
         else:
             return []
diff --git a/bdfr/downloader.py b/bdfr/downloader.py
index fa5d10c..1cb6d46 100644
--- a/bdfr/downloader.py
+++ b/bdfr/downloader.py
@@ -8,6 +8,7 @@ import time
 from datetime import datetime
 from multiprocessing import Pool
 from pathlib import Path
+from time import sleep
 
 import praw
 import praw.exceptions
@@ -42,11 +43,16 @@ class RedditDownloader(RedditConnector):
 
     def download(self):
         for generator in self.reddit_lists:
-            for submission in generator:
-                try:
-                    self._download_submission(submission)
-                except prawcore.PrawcoreException as e:
-                    logger.error(f"Submission {submission.id} failed to download due to a PRAW exception: {e}")
+            try:
+                for submission in generator:
+                    try:
+                        self._download_submission(submission)
+                    except prawcore.PrawcoreException as e:
+                        logger.error(f"Submission {submission.id} failed to download due to a PRAW exception: {e}")
+            except prawcore.PrawcoreException as e:
+                logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
+                logger.debug("Waiting 60 seconds to continue")
+                sleep(60)
 
     def _download_submission(self, submission: praw.models.Submission):
         if submission.id in self.excluded_submission_ids: