ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add 5xx soft fail for clone/archive

Browse Source
This commit is contained in:
OMEGARAZER
2022-12-10 21:19:29 -05:00
parent 3aa740e979
commit ac91c9089c
2 changed files with 35 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
bdfr/archiver.py
View File

@@ -4,6 +4,7 @@
import json import json
import logging import logging
import re import re
from time import sleep
from typing import Iterator, Union from typing import Iterator, Union
import dict2xml import dict2xml
@@ -28,23 +29,28 @@ class Archiver(RedditConnector):
def download(self): def download(self):
for generator in self.reddit_lists: for generator in self.reddit_lists:
for submission in generator: try:
try: for submission in generator:
if (submission.author and submission.author.name in self.args.ignore_user) or ( try:
submission.author is None and "DELETED" in self.args.ignore_user if (submission.author and submission.author.name in self.args.ignore_user) or (
): submission.author is None and "DELETED" in self.args.ignore_user
logger.debug( ):
f"Submission {submission.id} in {submission.subreddit.display_name} skipped" logger.debug(
f" due to {submission.author.name if submission.author else 'DELETED'} being an ignored user" f"Submission {submission.id} in {submission.subreddit.display_name} skipped due to"
) f" {submission.author.name if submission.author else 'DELETED'} being an ignored user"
continue )
if submission.id in self.excluded_submission_ids: continue
logger.debug(f"Object {submission.id} in exclusion list, skipping") if submission.id in self.excluded_submission_ids:
continue logger.debug(f"Object {submission.id} in exclusion list, skipping")
logger.debug(f"Attempting to archive submission {submission.id}") continue
self.write_entry(submission) logger.debug(f"Attempting to archive submission {submission.id}")
except prawcore.PrawcoreException as e: self.write_entry(submission)
logger.error(f"Submission {submission.id} failed to be archived due to a PRAW exception: {e}") except prawcore.PrawcoreException as e:
logger.error(f"Submission {submission.id} failed to be archived due to a PRAW exception: {e}")
except prawcore.PrawcoreException as e:
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
sleep(60)
def get_submissions_from_link(self) -> list[list[praw.models.Submission]]: def get_submissions_from_link(self) -> list[list[praw.models.Submission]]:
supplied_submissions = [] supplied_submissions = []

18
bdfr/cloner.py
View File

@@ -2,6 +2,7 @@
# coding=utf-8 # coding=utf-8
import logging import logging
from time import sleep
import prawcore import prawcore
@@ -18,9 +19,14 @@ class RedditCloner(RedditDownloader, Archiver):
def download(self): def download(self):
for generator in self.reddit_lists: for generator in self.reddit_lists:
for submission in generator: try:
try: for submission in generator:
self._download_submission(submission) try:
self.write_entry(submission) self._download_submission(submission)
except prawcore.PrawcoreException as e: self.write_entry(submission)
logger.error(f"Submission {submission.id} failed to be cloned due to a PRAW exception: {e}") except prawcore.PrawcoreException as e:
logger.error(f"Submission {submission.id} failed to be cloned due to a PRAW exception: {e}")
except prawcore.PrawcoreException as e:
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
sleep(60)