ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add soft fail on 5xx Prawcore errors.

Browse Source
This commit is contained in:
OMEGARAZER
2022-12-10 12:36:54 -05:00
parent 1bc20f238e
commit 3aa740e979
2 changed files with 36 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
bdfr/connector.py
View File

@@ -13,6 +13,7 @@ from abc import ABCMeta, abstractmethod
from datetime import datetime from datetime import datetime
from enum import Enum, auto from enum import Enum, auto
from pathlib import Path from pathlib import Path
from time import sleep
from typing import Callable, Iterator from typing import Callable, Iterator
import appdirs import appdirs
@@ -353,26 +354,31 @@ class RedditConnector(metaclass=ABCMeta):
generators = [] generators = []
for user in self.args.user: for user in self.args.user:
try: try:
self.check_user_existence(user) try:
except errors.BulkDownloaderException as e: self.check_user_existence(user)
logger.error(e) except errors.BulkDownloaderException as e:
continue logger.error(e)
if self.args.submitted: continue
logger.debug(f"Retrieving submitted posts of user {self.args.user}") if self.args.submitted:
generators.append( logger.debug(f"Retrieving submitted posts of user {user}")
self.create_filtered_listing_generator( generators.append(
self.reddit_instance.redditor(user).submissions, self.create_filtered_listing_generator(
self.reddit_instance.redditor(user).submissions,
)
) )
) if not self.authenticated and any((self.args.upvoted, self.args.saved)):
if not self.authenticated and any((self.args.upvoted, self.args.saved)): logger.warning("Accessing user lists requires authentication")
logger.warning("Accessing user lists requires authentication") else:
else: if self.args.upvoted:
if self.args.upvoted: logger.debug(f"Retrieving upvoted posts of user {user}")
logger.debug(f"Retrieving upvoted posts of user {self.args.user}") generators.append(self.reddit_instance.redditor(user).upvoted(limit=self.args.limit))
generators.append(self.reddit_instance.redditor(user).upvoted(limit=self.args.limit)) if self.args.saved:
if self.args.saved: logger.debug(f"Retrieving saved posts of user {user}")
logger.debug(f"Retrieving saved posts of user {self.args.user}") generators.append(self.reddit_instance.redditor(user).saved(limit=self.args.limit))
generators.append(self.reddit_instance.redditor(user).saved(limit=self.args.limit)) except prawcore.PrawcoreException as e:
logger.error(f"User {user} failed to be retrieved due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
sleep(60)
return generators return generators
else: else:
return [] return []

16
bdfr/downloader.py
View File

@@ -8,6 +8,7 @@ import time
from datetime import datetime from datetime import datetime
from multiprocessing import Pool from multiprocessing import Pool
from pathlib import Path from pathlib import Path
from time import sleep
import praw import praw
import praw.exceptions import praw.exceptions
@@ -42,11 +43,16 @@ class RedditDownloader(RedditConnector):
def download(self): def download(self):
for generator in self.reddit_lists: for generator in self.reddit_lists:
for submission in generator: try:
try: for submission in generator:
self._download_submission(submission) try:
except prawcore.PrawcoreException as e: self._download_submission(submission)
logger.error(f"Submission {submission.id} failed to download due to a PRAW exception: {e}") except prawcore.PrawcoreException as e:
logger.error(f"Submission {submission.id} failed to download due to a PRAW exception: {e}")
except prawcore.PrawcoreException as e:
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
sleep(60)
def _download_submission(self, submission: praw.models.Submission): def _download_submission(self, submission: praw.models.Submission):
if submission.id in self.excluded_submission_ids: if submission.id in self.excluded_submission_ids: