ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add option to skip specified subreddits (#268)

Browse Source 
* Rename variables

* Add option to skip specific subreddits

* Update README
This commit is contained in:
Serene
2021-04-17 19:56:43 +10:00
committed by Ali Parlakci
parent c85ae3fc69
commit d8752b15fa
5 changed files with 33 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
bdfr/downloader.py
View File

@@ -93,6 +93,9 @@ class RedditDownloader:
self.authenticator = self._create_authenticator()
logger.log(9, 'Created site authenticator')
self.args.skip_subreddit = self._split_args_input(self.args.skip_subreddit)
self.args.skip_subreddit = set([sub.lower() for sub in self.args.skip_subreddit])
def _read_config(self):
"""Read any cfg values that need to be processed"""
if self.args.max_wait_time is None:
@@ -210,13 +213,13 @@ class RedditDownloader:
return match.group(1)
@staticmethod
def _split_args_input(subreddit_entries: list[str]) -> set[str]:
all_subreddits = []
def _split_args_input(entries: list[str]) -> set[str]:
all_entries = []
split_pattern = re.compile(r'[,;]\s?')
for entry in subreddit_entries:
for entry in entries:
results = re.split(split_pattern, entry)
all_subreddits.extend([RedditDownloader._sanitise_subreddit_name(name) for name in results])
return set(all_subreddits)
all_entries.extend([RedditDownloader._sanitise_subreddit_name(name) for name in results])
return set(all_entries)
def _get_subreddits(self) -> list[praw.models.ListingGenerator]:
if self.args.subreddit:
@@ -354,8 +357,10 @@ class RedditDownloader:
for generator in self.reddit_lists:
for submission in generator:
if submission.id in self.excluded_submission_ids:
logger.debug(f'Submission {submission.id} in exclusion list, skipping')
logger.debug(f'Object {submission.id} in exclusion list, skipping')
continue
elif submission.subreddit.display_name.lower() in self.args.skip_subreddit:
logger.debug(f'Submission {submission.id} in {submission.subreddit.display_name} in skip list')
else:
logger.debug(f'Attempting to download submission {submission.id}')
self._download_submission(submission)