From a8c213627077c3e7464daa6ad92e84659bc54e98 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Sun, 2 May 2021 19:48:25 +1000 Subject: [PATCH] Add fallback downloader --- bdfr/site_downloaders/download_factory.py | 3 ++ .../fallback_downloaders/__init__.py | 0 .../fallback_downloader.py | 15 +++++++ .../youtubedl_fallback.py | 40 +++++++++++++++++++ .../fallback_downloaders/__init__.py | 0 .../youtubedl_fallback.py | 36 +++++++++++++++++ .../site_downloaders/test_download_factory.py | 6 ++- 7 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 bdfr/site_downloaders/fallback_downloaders/__init__.py create mode 100644 bdfr/site_downloaders/fallback_downloaders/fallback_downloader.py create mode 100644 bdfr/site_downloaders/fallback_downloaders/youtubedl_fallback.py create mode 100644 tests/site_downloaders/fallback_downloaders/__init__.py create mode 100644 tests/site_downloaders/fallback_downloaders/youtubedl_fallback.py diff --git a/bdfr/site_downloaders/download_factory.py b/bdfr/site_downloaders/download_factory.py index 8ee4d26..157814f 100644 --- a/bdfr/site_downloaders/download_factory.py +++ b/bdfr/site_downloaders/download_factory.py @@ -9,6 +9,7 @@ from bdfr.exceptions import NotADownloadableLinkError from bdfr.site_downloaders.base_downloader import BaseDownloader from bdfr.site_downloaders.direct import Direct from bdfr.site_downloaders.erome import Erome +from bdfr.site_downloaders.fallback_downloaders.youtubedl_fallback import YoutubeDlFallback from bdfr.site_downloaders.gallery import Gallery from bdfr.site_downloaders.gfycat import Gfycat from bdfr.site_downloaders.imgur import Imgur @@ -47,6 +48,8 @@ class DownloadFactory: return Streamable elif re.match(r'i\.redd\.it.*', sanitised_url): return Direct + elif YoutubeDlFallback.can_handle_link(sanitised_url): + return YoutubeDlFallback else: raise NotADownloadableLinkError( f'No downloader module exists for url {url}') diff --git a/bdfr/site_downloaders/fallback_downloaders/__init__.py b/bdfr/site_downloaders/fallback_downloaders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bdfr/site_downloaders/fallback_downloaders/fallback_downloader.py b/bdfr/site_downloaders/fallback_downloaders/fallback_downloader.py new file mode 100644 index 0000000..deeb213 --- /dev/null +++ b/bdfr/site_downloaders/fallback_downloaders/fallback_downloader.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +from abc import ABC, abstractmethod + +from bdfr.site_downloaders.base_downloader import BaseDownloader + + +class BaseFallbackDownloader(BaseDownloader, ABC): + + @staticmethod + @abstractmethod + def can_handle_link(url: str) -> bool: + """Returns whether the fallback downloader can download this link""" + raise NotImplementedError diff --git a/bdfr/site_downloaders/fallback_downloaders/youtubedl_fallback.py b/bdfr/site_downloaders/fallback_downloaders/youtubedl_fallback.py new file mode 100644 index 0000000..6e006ec --- /dev/null +++ b/bdfr/site_downloaders/fallback_downloaders/youtubedl_fallback.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +import logging +from typing import Optional + +import youtube_dl +from praw.models import Submission + +from bdfr.resource import Resource +from bdfr.site_authenticator import SiteAuthenticator +from bdfr.site_downloaders.fallback_downloaders.fallback_downloader import BaseFallbackDownloader +from bdfr.site_downloaders.youtube import Youtube + +logger = logging.getLogger(__name__) + + +class YoutubeDlFallback(BaseFallbackDownloader, Youtube): + def __init__(self, post: Submission): + super(YoutubeDlFallback, self).__init__(post) + + def find_resources(self, authenticator: Optional[SiteAuthenticator] = None) -> list[Resource]: + out = super()._download_video({}) + return [out] + + @staticmethod + def can_handle_link(url: str) -> bool: + yt_logger = logging.getLogger('youtube-dl') + yt_logger.setLevel(logging.CRITICAL) + with youtube_dl.YoutubeDL({ + 'logger': yt_logger, + }) as ydl: + try: + result = ydl.extract_info(url, download=False) + if result: + return True + except youtube_dl.DownloadError as e: + logger.exception(e) + return False + return False diff --git a/tests/site_downloaders/fallback_downloaders/__init__.py b/tests/site_downloaders/fallback_downloaders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/site_downloaders/fallback_downloaders/youtubedl_fallback.py b/tests/site_downloaders/fallback_downloaders/youtubedl_fallback.py new file mode 100644 index 0000000..7f393b6 --- /dev/null +++ b/tests/site_downloaders/fallback_downloaders/youtubedl_fallback.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +from unittest.mock import MagicMock + +import pytest + +from bdfr.resource import Resource +from bdfr.site_downloaders.fallback_downloaders.youtubedl_fallback import YoutubeDlFallback + + +@pytest.mark.online +@pytest.mark.parametrize(('test_url', 'expected'), ( + ('https://www.reddit.com/r/specializedtools/comments/n2nw5m/bamboo_splitter/', True), + ('https://www.youtube.com/watch?v=P19nvJOmqCc', True), + ('https://www.example.com/test', False), +)) +def test_can_handle_link(test_url: str, expected: bool): + result = YoutubeDlFallback.can_handle_link(test_url) + assert result == expected + + +@pytest.mark.online +@pytest.mark.slow +@pytest.mark.parametrize(('test_url', 'expected_hash'), ( + ('https://streamable.com/dt46y', '1e7f4928e55de6e3ca23d85cc9246bbb'), + ('https://streamable.com/t8sem', '49b2d1220c485455548f1edbc05d4ecf'), + ('https://www.reddit.com/r/specializedtools/comments/n2nw5m/bamboo_splitter/', '21968d3d92161ea5e0abdcaf6311b06c'), +)) +def test_find_resources(test_url: str, expected_hash: str): + test_submission = MagicMock() + test_submission.url = test_url + downloader = YoutubeDlFallback(test_submission) + resources = downloader.find_resources() + assert len(resources) == 1 + assert isinstance(resources[0], Resource) + assert resources[0].hash.hexdigest() == expected_hash diff --git a/tests/site_downloaders/test_download_factory.py b/tests/site_downloaders/test_download_factory.py index aab8540..005974c 100644 --- a/tests/site_downloaders/test_download_factory.py +++ b/tests/site_downloaders/test_download_factory.py @@ -9,6 +9,7 @@ from bdfr.site_downloaders.base_downloader import BaseDownloader from bdfr.site_downloaders.direct import Direct from bdfr.site_downloaders.download_factory import DownloadFactory from bdfr.site_downloaders.erome import Erome +from bdfr.site_downloaders.fallback_downloaders.youtubedl_fallback import YoutubeDlFallback from bdfr.site_downloaders.gallery import Gallery from bdfr.site_downloaders.gfycat import Gfycat from bdfr.site_downloaders.imgur import Imgur @@ -19,6 +20,7 @@ from bdfr.site_downloaders.vreddit import VReddit from bdfr.site_downloaders.youtube import Youtube +@pytest.mark.online @pytest.mark.parametrize(('test_submission_url', 'expected_class'), ( ('https://v.redd.it/9z1dnk3xr5k61', VReddit), ('https://www.reddit.com/r/TwoXChromosomes/comments/lu29zn/i_refuse_to_live_my_life' @@ -41,7 +43,9 @@ from bdfr.site_downloaders.youtube import Youtube ('https://i.imgur.com/3SKrQfK.jpg?1', Direct), ('https://dynasty-scans.com/system/images_images/000/017/819/original/80215103_p0.png?1612232781', Direct), ('https://m.imgur.com/a/py3RW0j', Imgur), - ('https://streamable.com/dt46y', Streamable) + ('https://streamable.com/dt46y', Streamable), + ('https://vimeo.com/channels/31259/53576664', YoutubeDlFallback), + ('http://video.pbs.org/viralplayer/2365173446/', YoutubeDlFallback), )) def test_factory_lever_good(test_submission_url: str, expected_class: BaseDownloader, reddit_instance: praw.Reddit): result = DownloadFactory.pull_lever(test_submission_url)