ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #695 from Serene-Arc/bug_fix_677

Browse Source 
Closes https://github.com/aliparlakci/bulk-downloader-for-reddit/issues/677
This commit is contained in:
Serene
2022-11-21 14:43:24 +10:00
committed by GitHub
parent 49727aea6e 1385545e26
commit 9ba62f8c97
3 changed files with 22 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
bdfr/archiver.py
View File

@@ -8,6 +8,7 @@ from typing import Iterator
import dict2xml import dict2xml
import praw.models import praw.models
import prawcore
import yaml import yaml
from bdfr.archive_entry.base_archive_entry import BaseArchiveEntry from bdfr.archive_entry.base_archive_entry import BaseArchiveEntry
@@ -28,17 +29,20 @@ class Archiver(RedditConnector):
def download(self): def download(self):
for generator in self.reddit_lists: for generator in self.reddit_lists:
for submission in generator: for submission in generator:
if (submission.author and submission.author.name in self.args.ignore_user) or \ try:
(submission.author is None and 'DELETED' in self.args.ignore_user): if (submission.author and submission.author.name in self.args.ignore_user) or \
logger.debug( (submission.author is None and 'DELETED' in self.args.ignore_user):
f'Submission {submission.id} in {submission.subreddit.display_name} skipped' logger.debug(
f' due to {submission.author.name if submission.author else "DELETED"} being an ignored user') f'Submission {submission.id} in {submission.subreddit.display_name} skipped'
continue f' due to {submission.author.name if submission.author else "DELETED"} being an ignored user')
if submission.id in self.excluded_submission_ids: continue
logger.debug(f'Object {submission.id} in exclusion list, skipping') if submission.id in self.excluded_submission_ids:
continue logger.debug(f'Object {submission.id} in exclusion list, skipping')
logger.debug(f'Attempting to archive submission {submission.id}') continue
self.write_entry(submission) logger.debug(f'Attempting to archive submission {submission.id}')
self.write_entry(submission)
except prawcore.PrawcoreException as e:
logger.error(f'Submission {submission.id} failed to be archived due to a PRAW exception: {e}')
def get_submissions_from_link(self) -> list[list[praw.models.Submission]]: def get_submissions_from_link(self) -> list[list[praw.models.Submission]]:
supplied_submissions = [] supplied_submissions = []

6
bdfr/downloader.py
View File

@@ -12,6 +12,7 @@ from pathlib import Path
import praw import praw
import praw.exceptions import praw.exceptions
import praw.models import praw.models
import prawcore
from bdfr import exceptions as errors from bdfr import exceptions as errors
from bdfr.configuration import Configuration from bdfr.configuration import Configuration
@@ -42,7 +43,10 @@ class RedditDownloader(RedditConnector):
def download(self): def download(self):
for generator in self.reddit_lists: for generator in self.reddit_lists:
for submission in generator: for submission in generator:
self._download_submission(submission) try:
self._download_submission(submission)
except prawcore.PrawcoreException as e:
logger.error(f'Submission {submission.id} failed to download due to a PRAW exception: {e}')
def _download_submission(self, submission: praw.models.Submission): def _download_submission(self, submission: praw.models.Submission):
if submission.id in self.excluded_submission_ids: if submission.id in self.excluded_submission_ids:

2
tests/integration_tests/test_download_integration.py
View File

@@ -232,6 +232,8 @@ def test_cli_download_long(test_args: list[str], tmp_path: Path):
['--subreddit', 'donaldtrump', '-L', 10], # Banned subreddit ['--subreddit', 'donaldtrump', '-L', 10], # Banned subreddit
['--user', 'djnish', '--user', 'helen_darten', '-m', 'cuteanimalpics', '-L', 10], ['--user', 'djnish', '--user', 'helen_darten', '-m', 'cuteanimalpics', '-L', 10],
['--subreddit', 'friends', '-L', 10], ['--subreddit', 'friends', '-L', 10],
['-l', 'ijy4ch'],
['-l', 'kw4wjm'],
)) ))
def test_cli_download_soft_fail(test_args: list[str], tmp_path: Path): def test_cli_download_soft_fail(test_args: list[str], tmp_path: Path):
runner = CliRunner() runner = CliRunner()