ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add catch for per-submission praw errors

Browse Source
This commit is contained in:
Serene-Arc
2022-11-20 18:54:56 +10:00
parent 49727aea6e
commit 5341d6f12c
2 changed files with 20 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
bdfr/archiver.py
View File

@@ -8,6 +8,7 @@ from typing import Iterator
import dict2xml import dict2xml
import praw.models import praw.models
import prawcore
import yaml import yaml
from bdfr.archive_entry.base_archive_entry import BaseArchiveEntry from bdfr.archive_entry.base_archive_entry import BaseArchiveEntry
@@ -28,17 +29,20 @@ class Archiver(RedditConnector):
def download(self): def download(self):
for generator in self.reddit_lists: for generator in self.reddit_lists:
for submission in generator: for submission in generator:
if (submission.author and submission.author.name in self.args.ignore_user) or \ try:
(submission.author is None and 'DELETED' in self.args.ignore_user): if (submission.author and submission.author.name in self.args.ignore_user) or \
logger.debug( (submission.author is None and 'DELETED' in self.args.ignore_user):
f'Submission {submission.id} in {submission.subreddit.display_name} skipped' logger.debug(
f' due to {submission.author.name if submission.author else "DELETED"} being an ignored user') f'Submission {submission.id} in {submission.subreddit.display_name} skipped'
continue f' due to {submission.author.name if submission.author else "DELETED"} being an ignored user')
if submission.id in self.excluded_submission_ids: continue
logger.debug(f'Object {submission.id} in exclusion list, skipping') if submission.id in self.excluded_submission_ids:
continue logger.debug(f'Object {submission.id} in exclusion list, skipping')
logger.debug(f'Attempting to archive submission {submission.id}') continue
self.write_entry(submission) logger.debug(f'Attempting to archive submission {submission.id}')
self.write_entry(submission)
except prawcore.PrawcoreException as e:
logger.error(f'Submission {submission.id} failed to be archived due to a PRAW exception: {e}')
def get_submissions_from_link(self) -> list[list[praw.models.Submission]]: def get_submissions_from_link(self) -> list[list[praw.models.Submission]]:
supplied_submissions = [] supplied_submissions = []

6
bdfr/downloader.py
View File

@@ -12,6 +12,7 @@ from pathlib import Path
import praw import praw
import praw.exceptions import praw.exceptions
import praw.models import praw.models
import prawcore
from bdfr import exceptions as errors from bdfr import exceptions as errors
from bdfr.configuration import Configuration from bdfr.configuration import Configuration
@@ -42,7 +43,10 @@ class RedditDownloader(RedditConnector):
def download(self): def download(self):
for generator in self.reddit_lists: for generator in self.reddit_lists:
for submission in generator: for submission in generator:
self._download_submission(submission) try:
self._download_submission(submission)
except prawcore.PrawcoreException as e:
logger.error(f'Submission {submission.id} failed to download due to a PRAW exception: {e}')
def _download_submission(self, submission: praw.models.Submission): def _download_submission(self, submission: praw.models.Submission):
if submission.id in self.excluded_submission_ids: if submission.id in self.excluded_submission_ids: