ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add 5xx soft fail for clone/archive

Browse Source
This commit is contained in:
OMEGARAZER
2022-12-10 21:19:29 -05:00
parent 3aa740e979
commit ac91c9089c
2 changed files with 35 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
bdfr/cloner.py
View File

@@ -2,6 +2,7 @@
# coding=utf-8
import logging
from time import sleep
import prawcore
@@ -18,9 +19,14 @@ class RedditCloner(RedditDownloader, Archiver):
def download(self):
for generator in self.reddit_lists:
for submission in generator:
try:
self._download_submission(submission)
self.write_entry(submission)
except prawcore.PrawcoreException as e:
logger.error(f"Submission {submission.id} failed to be cloned due to a PRAW exception: {e}")
try:
for submission in generator:
try:
self._download_submission(submission)
self.write_entry(submission)
except prawcore.PrawcoreException as e:
logger.error(f"Submission {submission.id} failed to be cloned due to a PRAW exception: {e}")
except prawcore.PrawcoreException as e:
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
sleep(60)