ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add soft fail on 5xx Prawcore errors.

Browse Source
This commit is contained in:
OMEGARAZER
2022-12-10 12:36:54 -05:00
parent 1bc20f238e
commit 3aa740e979
2 changed files with 36 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
bdfr/downloader.py
View File

@@ -8,6 +8,7 @@ import time
from datetime import datetime
from multiprocessing import Pool
from pathlib import Path
from time import sleep
import praw
import praw.exceptions
@@ -42,11 +43,16 @@ class RedditDownloader(RedditConnector):
def download(self):
for generator in self.reddit_lists:
for submission in generator:
try:
self._download_submission(submission)
except prawcore.PrawcoreException as e:
logger.error(f"Submission {submission.id} failed to download due to a PRAW exception: {e}")
try:
for submission in generator:
try:
self._download_submission(submission)
except prawcore.PrawcoreException as e:
logger.error(f"Submission {submission.id} failed to download due to a PRAW exception: {e}")
except prawcore.PrawcoreException as e:
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
sleep(60)
def _download_submission(self, submission: praw.models.Submission):
if submission.id in self.excluded_submission_ids: