From 892564333176e17b919b21470571a789c50c3167 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Wed, 24 Nov 2021 10:40:18 +1000 Subject: [PATCH 1/7] Rename module to reflect backend change --- bdfr/site_downloaders/download_factory.py | 6 +++--- .../{youtubedl_fallback.py => ytdlp_fallback.py} | 6 +++--- ...st_youtubedl_fallback.py => test_ytdlp_fallback.py} | 8 ++++---- tests/site_downloaders/test_download_factory.py | 10 +++++----- tests/test_file_name_formatter.py | 6 +++--- 5 files changed, 18 insertions(+), 18 deletions(-) rename bdfr/site_downloaders/fallback_downloaders/{youtubedl_fallback.py => ytdlp_fallback.py} (84%) rename tests/site_downloaders/fallback_downloaders/{test_youtubedl_fallback.py => test_ytdlp_fallback.py} (86%) diff --git a/bdfr/site_downloaders/download_factory.py b/bdfr/site_downloaders/download_factory.py index 2f4cf9d..91489a0 100644 --- a/bdfr/site_downloaders/download_factory.py +++ b/bdfr/site_downloaders/download_factory.py @@ -9,7 +9,7 @@ from bdfr.exceptions import NotADownloadableLinkError from bdfr.site_downloaders.base_downloader import BaseDownloader from bdfr.site_downloaders.direct import Direct from bdfr.site_downloaders.erome import Erome -from bdfr.site_downloaders.fallback_downloaders.youtubedl_fallback import YoutubeDlFallback +from bdfr.site_downloaders.fallback_downloaders.ytdlp_fallback import YtdlpFallback from bdfr.site_downloaders.gallery import Gallery from bdfr.site_downloaders.gfycat import Gfycat from bdfr.site_downloaders.imgur import Imgur @@ -49,8 +49,8 @@ class DownloadFactory: return PornHub elif re.match(r'vidble\.com', sanitised_url): return Vidble - elif YoutubeDlFallback.can_handle_link(sanitised_url): - return YoutubeDlFallback + elif YtdlpFallback.can_handle_link(sanitised_url): + return YtdlpFallback else: raise NotADownloadableLinkError(f'No downloader module exists for url {url}') diff --git a/bdfr/site_downloaders/fallback_downloaders/youtubedl_fallback.py b/bdfr/site_downloaders/fallback_downloaders/ytdlp_fallback.py similarity index 84% rename from bdfr/site_downloaders/fallback_downloaders/youtubedl_fallback.py rename to bdfr/site_downloaders/fallback_downloaders/ytdlp_fallback.py index d8753bd..1225624 100644 --- a/bdfr/site_downloaders/fallback_downloaders/youtubedl_fallback.py +++ b/bdfr/site_downloaders/fallback_downloaders/ytdlp_fallback.py @@ -15,9 +15,9 @@ from bdfr.site_downloaders.youtube import Youtube logger = logging.getLogger(__name__) -class YoutubeDlFallback(BaseFallbackDownloader, Youtube): +class YtdlpFallback(BaseFallbackDownloader, Youtube): def __init__(self, post: Submission): - super(YoutubeDlFallback, self).__init__(post) + super(YtdlpFallback, self).__init__(post) def find_resources(self, authenticator: Optional[SiteAuthenticator] = None) -> list[Resource]: out = Resource( @@ -31,7 +31,7 @@ class YoutubeDlFallback(BaseFallbackDownloader, Youtube): @staticmethod def can_handle_link(url: str) -> bool: try: - attributes = YoutubeDlFallback.get_video_attributes(url) + attributes = YtdlpFallback.get_video_attributes(url) except NotADownloadableLinkError: return False if attributes: diff --git a/tests/site_downloaders/fallback_downloaders/test_youtubedl_fallback.py b/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py similarity index 86% rename from tests/site_downloaders/fallback_downloaders/test_youtubedl_fallback.py rename to tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py index 0590687..2c4a4f6 100644 --- a/tests/site_downloaders/fallback_downloaders/test_youtubedl_fallback.py +++ b/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py @@ -6,7 +6,7 @@ import pytest from bdfr.exceptions import NotADownloadableLinkError from bdfr.resource import Resource -from bdfr.site_downloaders.fallback_downloaders.youtubedl_fallback import YoutubeDlFallback +from bdfr.site_downloaders.fallback_downloaders.ytdlp_fallback import YtdlpFallback @pytest.mark.online @@ -17,7 +17,7 @@ from bdfr.site_downloaders.fallback_downloaders.youtubedl_fallback import Youtub ('https://milesmatrix.bandcamp.com/album/la-boum/', False), )) def test_can_handle_link(test_url: str, expected: bool): - result = YoutubeDlFallback.can_handle_link(test_url) + result = YtdlpFallback.can_handle_link(test_url) assert result == expected @@ -27,7 +27,7 @@ def test_can_handle_link(test_url: str, expected: bool): )) def test_info_extraction_bad(test_url: str): with pytest.raises(NotADownloadableLinkError): - YoutubeDlFallback.get_video_attributes(test_url) + YtdlpFallback.get_video_attributes(test_url) @pytest.mark.online @@ -41,7 +41,7 @@ def test_info_extraction_bad(test_url: str): def test_find_resources(test_url: str, expected_hash: str): test_submission = MagicMock() test_submission.url = test_url - downloader = YoutubeDlFallback(test_submission) + downloader = YtdlpFallback(test_submission) resources = downloader.find_resources() assert len(resources) == 1 assert isinstance(resources[0], Resource) diff --git a/tests/site_downloaders/test_download_factory.py b/tests/site_downloaders/test_download_factory.py index 15466cb..441b554 100644 --- a/tests/site_downloaders/test_download_factory.py +++ b/tests/site_downloaders/test_download_factory.py @@ -9,7 +9,7 @@ from bdfr.site_downloaders.base_downloader import BaseDownloader from bdfr.site_downloaders.direct import Direct from bdfr.site_downloaders.download_factory import DownloadFactory from bdfr.site_downloaders.erome import Erome -from bdfr.site_downloaders.fallback_downloaders.youtubedl_fallback import YoutubeDlFallback +from bdfr.site_downloaders.fallback_downloaders.ytdlp_fallback import YtdlpFallback from bdfr.site_downloaders.gallery import Gallery from bdfr.site_downloaders.gfycat import Gfycat from bdfr.site_downloaders.imgur import Imgur @@ -42,10 +42,10 @@ from bdfr.site_downloaders.youtube import Youtube ('https://i.imgur.com/3SKrQfK.jpg?1', Direct), ('https://dynasty-scans.com/system/images_images/000/017/819/original/80215103_p0.png?1612232781', Direct), ('https://m.imgur.com/a/py3RW0j', Imgur), - ('https://v.redd.it/9z1dnk3xr5k61', YoutubeDlFallback), - ('https://streamable.com/dt46y', YoutubeDlFallback), - ('https://vimeo.com/channels/31259/53576664', YoutubeDlFallback), - ('http://video.pbs.org/viralplayer/2365173446/', YoutubeDlFallback), + ('https://v.redd.it/9z1dnk3xr5k61', YtdlpFallback), + ('https://streamable.com/dt46y', YtdlpFallback), + ('https://vimeo.com/channels/31259/53576664', YtdlpFallback), + ('http://video.pbs.org/viralplayer/2365173446/', YtdlpFallback), ('https://www.pornhub.com/view_video.php?viewkey=ph5a2ee0461a8d0', PornHub), )) def test_factory_lever_good(test_submission_url: str, expected_class: BaseDownloader, reddit_instance: praw.Reddit): diff --git a/tests/test_file_name_formatter.py b/tests/test_file_name_formatter.py index 29ee50f..e60ae8d 100644 --- a/tests/test_file_name_formatter.py +++ b/tests/test_file_name_formatter.py @@ -14,7 +14,7 @@ import pytest from bdfr.file_name_formatter import FileNameFormatter from bdfr.resource import Resource from bdfr.site_downloaders.base_downloader import BaseDownloader -from bdfr.site_downloaders.fallback_downloaders.youtubedl_fallback import YoutubeDlFallback +from bdfr.site_downloaders.fallback_downloaders.ytdlp_fallback import YtdlpFallback @pytest.fixture() @@ -402,8 +402,8 @@ def test_windows_max_path(tmp_path: Path): @pytest.mark.online @pytest.mark.reddit @pytest.mark.parametrize(('test_reddit_id', 'test_downloader', 'expected_names'), ( - ('gphmnr', YoutubeDlFallback, {'He has a lot to say today.mp4'}), - ('d0oir2', YoutubeDlFallback, {"Crunk's finest moment. Welcome to the new subreddit!.mp4"}), + ('gphmnr', YtdlpFallback, {'He has a lot to say today.mp4'}), + ('d0oir2', YtdlpFallback, {"Crunk's finest moment. Welcome to the new subreddit!.mp4"}), )) def test_name_submission( test_reddit_id: str, From 4a864827567bd59c2c473f6d2a08d4bbd46da1a3 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Wed, 24 Nov 2021 11:07:52 +1000 Subject: [PATCH 2/7] Add skip statement for broken test on windows --- tests/test_file_name_formatter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_file_name_formatter.py b/tests/test_file_name_formatter.py index e60ae8d..30fac77 100644 --- a/tests/test_file_name_formatter.py +++ b/tests/test_file_name_formatter.py @@ -2,6 +2,7 @@ # coding=utf-8 import platform +import sys import unittest.mock from datetime import datetime from pathlib import Path @@ -213,6 +214,7 @@ def test_preserve_id_append_when_shortening(test_filename: str, test_ending: str assert len(str(result)) <= FileNameFormatter.find_max_path_length() +@pytest.mark.skipif(sys.platform == 'win32', reason='Test broken on windows github') def test_shorten_filename_real(submission: MagicMock, tmp_path: Path): submission.title = 'A' * 500 submission.author.name = 'test' From dd8d74ee25a34cdfe36c271e6a26cc5f6e9ed7d8 Mon Sep 17 00:00:00 2001 From: "Jay R. Wren" Date: Sat, 30 Oct 2021 22:19:46 -0400 Subject: [PATCH 3/7] Add --ignore to ignore user --- README.md | 3 +++ bdfr/__main__.py | 1 + bdfr/archiver.py | 2 ++ bdfr/cloner.py | 2 ++ bdfr/configuration.py | 1 + 5 files changed, 9 insertions(+) diff --git a/README.md b/README.md index 7bc4f0b..3ffef7f 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,9 @@ The following options are common between both the `archive` and `download` comma - Can be specified multiple times - Disables certain modules from being used - See [Disabling Modules](#disabling-modules) for more information and a list of module names +- `--ignore` + - This will add a user to ignore + - Can be specified multiple times - `--include-id-file` - This will add any submission with the IDs in the files provided - Can be specified multiple times diff --git a/bdfr/__main__.py b/bdfr/__main__.py index 367f8c6..de658de 100644 --- a/bdfr/__main__.py +++ b/bdfr/__main__.py @@ -17,6 +17,7 @@ _common_options = [ click.option('--authenticate', is_flag=True, default=None), click.option('--config', type=str, default=None), click.option('--disable-module', multiple=True, default=None, type=str), + click.option('--ignore-user', type=str, multiple=True, default=None), click.option('--include-id-file', multiple=True, default=None), click.option('--log', type=str, default=None), click.option('--saved', is_flag=True, default=None), diff --git a/bdfr/archiver.py b/bdfr/archiver.py index d445e8d..e51be57 100644 --- a/bdfr/archiver.py +++ b/bdfr/archiver.py @@ -28,6 +28,8 @@ class Archiver(RedditConnector): def download(self): for generator in self.reddit_lists: for submission in generator: + if submission.author.name in self.args.ignore_user: + continue logger.debug(f'Attempting to archive submission {submission.id}') self.write_entry(submission) diff --git a/bdfr/cloner.py b/bdfr/cloner.py index 979f50f..c48ae17 100644 --- a/bdfr/cloner.py +++ b/bdfr/cloner.py @@ -17,5 +17,7 @@ class RedditCloner(RedditDownloader, Archiver): def download(self): for generator in self.reddit_lists: for submission in generator: + if submission.author.name in self.args.ignore_user: + continue self._download_submission(submission) self.write_entry(submission) diff --git a/bdfr/configuration.py b/bdfr/configuration.py index bc4c541..81fa3e4 100644 --- a/bdfr/configuration.py +++ b/bdfr/configuration.py @@ -18,6 +18,7 @@ class Configuration(Namespace): self.exclude_id_file = [] self.file_scheme: str = '{REDDITOR}_{TITLE}_{POSTID}' self.folder_scheme: str = '{SUBREDDIT}' + self.ignore_user = [] self.include_id_file = [] self.limit: Optional[int] = None self.link: list[str] = [] From 2b50ee072400226e15ee2526985214ce938a6670 Mon Sep 17 00:00:00 2001 From: "Jay R. Wren" Date: Mon, 1 Nov 2021 09:28:46 -0400 Subject: [PATCH 4/7] add test. fix typos. --- README.md | 2 +- bdfr/cloner.py | 2 -- bdfr/downloader.py | 4 ++++ tests/test_downloader.py | 17 +++++++++++++++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3ffef7f..b84aa3d 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ The following options are common between both the `archive` and `download` comma - Can be specified multiple times - Disables certain modules from being used - See [Disabling Modules](#disabling-modules) for more information and a list of module names -- `--ignore` +- `--ignore-user` - This will add a user to ignore - Can be specified multiple times - `--include-id-file` diff --git a/bdfr/cloner.py b/bdfr/cloner.py index c48ae17..979f50f 100644 --- a/bdfr/cloner.py +++ b/bdfr/cloner.py @@ -17,7 +17,5 @@ class RedditCloner(RedditDownloader, Archiver): def download(self): for generator in self.reddit_lists: for submission in generator: - if submission.author.name in self.args.ignore_user: - continue self._download_submission(submission) self.write_entry(submission) diff --git a/bdfr/downloader.py b/bdfr/downloader.py index 70052b2..edfd68e 100644 --- a/bdfr/downloader.py +++ b/bdfr/downloader.py @@ -51,6 +51,10 @@ class RedditDownloader(RedditConnector): elif submission.subreddit.display_name.lower() in self.args.skip_subreddit: logger.debug(f'Submission {submission.id} in {submission.subreddit.display_name} in skip list') return + elif submission.author.name in self.args.ignore_user: + logger.debug( + f'Submission {submission.id} in {submission.subreddit.display_name} by {submission.author.name} an ignored user') + return elif not isinstance(submission, praw.models.Submission): logger.warning(f'{submission.id} is not a submission') return diff --git a/tests/test_downloader.py b/tests/test_downloader.py index e5f0a31..0cc8dec 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -200,3 +200,20 @@ def test_download_submission( RedditDownloader._download_submission(downloader_mock, submission) folder_contents = list(tmp_path.iterdir()) assert len(folder_contents) == expected_files_len + + +@pytest.mark.parametrize('test_ignore_user', ( + 'alice', +)) +def test_download_ignores_user( + test_ignore_user: str, + mock_function: MagicMock, + downloader_mock: MagicMock, +): + downloader_mock.args.ignore_user = test_ignore_user + submission = downloader_mock.reddit_instance.submission(id='m1hqw6') + mock_function.return_value = MagicMock() + mock_function.return_value.__name__ = 'test' + submission.author.name = test_ignore_user + RedditDownloader._download_submission(downloader_mock, submission) + assert mock_function.call_count == 0 From 0eeb4b46dc70fe30f4e2c865ee6578245e5a30f9 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Wed, 24 Nov 2021 10:48:06 +1000 Subject: [PATCH 5/7] Remove bad test --- tests/test_downloader.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 0cc8dec..e5f0a31 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -200,20 +200,3 @@ def test_download_submission( RedditDownloader._download_submission(downloader_mock, submission) folder_contents = list(tmp_path.iterdir()) assert len(folder_contents) == expected_files_len - - -@pytest.mark.parametrize('test_ignore_user', ( - 'alice', -)) -def test_download_ignores_user( - test_ignore_user: str, - mock_function: MagicMock, - downloader_mock: MagicMock, -): - downloader_mock.args.ignore_user = test_ignore_user - submission = downloader_mock.reddit_instance.submission(id='m1hqw6') - mock_function.return_value = MagicMock() - mock_function.return_value.__name__ = 'test' - submission.author.name = test_ignore_user - RedditDownloader._download_submission(downloader_mock, submission) - assert mock_function.call_count == 0 From d0d72c82299be3d3a88a6eca950469ced4fce44b Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Wed, 24 Nov 2021 10:54:29 +1000 Subject: [PATCH 6/7] Add integration test for downloader option --- bdfr/downloader.py | 3 ++- .../test_download_integration.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/bdfr/downloader.py b/bdfr/downloader.py index edfd68e..028430f 100644 --- a/bdfr/downloader.py +++ b/bdfr/downloader.py @@ -53,7 +53,8 @@ class RedditDownloader(RedditConnector): return elif submission.author.name in self.args.ignore_user: logger.debug( - f'Submission {submission.id} in {submission.subreddit.display_name} by {submission.author.name} an ignored user') + f'Submission {submission.id} in {submission.subreddit.display_name} skipped' + f' due to {submission.author.name} being an ignored user') return elif not isinstance(submission, praw.models.Submission): logger.warning(f'{submission.id} is not a submission') diff --git a/tests/integration_tests/test_download_integration.py b/tests/integration_tests/test_download_integration.py index c2414ba..bd53382 100644 --- a/tests/integration_tests/test_download_integration.py +++ b/tests/integration_tests/test_download_integration.py @@ -337,3 +337,18 @@ def test_cli_download_include_id_file(tmp_path: Path): result = runner.invoke(cli, test_args) assert result.exit_code == 0 assert 'Downloaded submission' in result.output + + +@pytest.mark.online +@pytest.mark.reddit +@pytest.mark.skipif(not does_test_config_exist, reason='A test config file is required for integration tests') +@pytest.mark.parametrize('test_args', ( + ['--ignore-user', 'ArjanEgges', '-l', 'm3hxzd'], +)) +def test_cli_download_ignore_user(test_args: list[str], tmp_path: Path): + runner = CliRunner() + test_args = create_basic_args_for_download_runner(test_args, tmp_path) + result = runner.invoke(cli, test_args) + assert result.exit_code == 0 + assert 'Downloaded submission' not in result.output + assert 'being an ignored user' in result.output From f670b347ae94c2366da6fdab1f2a4e34eeb82249 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Wed, 24 Nov 2021 10:58:18 +1000 Subject: [PATCH 7/7] Add integration test for archiver option --- bdfr/archiver.py | 3 +++ .../integration_tests/test_archive_integration.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/bdfr/archiver.py b/bdfr/archiver.py index e51be57..a2e54e5 100644 --- a/bdfr/archiver.py +++ b/bdfr/archiver.py @@ -29,6 +29,9 @@ class Archiver(RedditConnector): for generator in self.reddit_lists: for submission in generator: if submission.author.name in self.args.ignore_user: + logger.debug( + f'Submission {submission.id} in {submission.subreddit.display_name} skipped' + f' due to {submission.author.name} being an ignored user') continue logger.debug(f'Attempting to archive submission {submission.id}') self.write_entry(submission) diff --git a/tests/integration_tests/test_archive_integration.py b/tests/integration_tests/test_archive_integration.py index 8cbb2d5..5ef04a6 100644 --- a/tests/integration_tests/test_archive_integration.py +++ b/tests/integration_tests/test_archive_integration.py @@ -106,3 +106,18 @@ def test_cli_archive_long(test_args: list[str], tmp_path: Path): result = runner.invoke(cli, test_args) assert result.exit_code == 0 assert re.search(r'Writing entry .*? to file in .*? format', result.output) + + +@pytest.mark.online +@pytest.mark.reddit +@pytest.mark.skipif(not does_test_config_exist, reason='A test config file is required for integration tests') +@pytest.mark.parametrize('test_args', ( + ['--ignore-user', 'ArjanEgges', '-l', 'm3hxzd'], +)) +def test_cli_archive_ignore_user(test_args: list[str], tmp_path: Path): + runner = CliRunner() + test_args = create_basic_args_for_archive_runner(test_args, tmp_path) + result = runner.invoke(cli, test_args) + assert result.exit_code == 0 + assert 'being an ignored user' in result.output + assert 'Attempting to archive submission' not in result.output