diff --git a/bdfr/site_downloaders/download_factory.py b/bdfr/site_downloaders/download_factory.py index 7035dc2..41813f9 100644 --- a/bdfr/site_downloaders/download_factory.py +++ b/bdfr/site_downloaders/download_factory.py @@ -21,10 +21,11 @@ from bdfr.site_downloaders.youtube import Youtube class DownloadFactory: @staticmethod def pull_lever(url: str) -> Type[BaseDownloader]: - sanitised_url = DownloadFactory._sanitise_url(url) + sanitised_url = DownloadFactory.sanitise_url(url) if re.match(r'(i\.)?imgur.*\.gifv$', sanitised_url): return Imgur - elif re.match(r'.*/.*\.\w{3,4}(\?[\w;&=]*)?$', sanitised_url): + elif re.match(r'.*/.*\.\w{3,4}(\?[\w;&=]*)?$', sanitised_url) and \ + not DownloadFactory.is_web_resource(sanitised_url): return Direct elif re.match(r'erome\.com.*', sanitised_url): return Erome @@ -49,9 +50,29 @@ class DownloadFactory: f'No downloader module exists for url {url}') @staticmethod - def _sanitise_url(url: str) -> str: + def sanitise_url(url: str) -> str: beginning_regex = re.compile(r'\s*(www\.?)?') split_url = urllib.parse.urlsplit(url) split_url = split_url.netloc + split_url.path split_url = re.sub(beginning_regex, '', split_url) return split_url + + @staticmethod + def is_web_resource(url: str) -> bool: + web_extensions = ( + 'asp', + 'aspx', + 'cfm', + 'cfml', + 'css', + 'htm', + 'html', + 'js', + 'php', + 'php3', + 'xhtml', + ) + if re.match(rf'(?i).*/.*\.({"|".join(web_extensions)})$', url): + return True + else: + return False diff --git a/tests/site_downloaders/test_download_factory.py b/tests/site_downloaders/test_download_factory.py index f02e9f7..4b5356c 100644 --- a/tests/site_downloaders/test_download_factory.py +++ b/tests/site_downloaders/test_download_factory.py @@ -69,6 +69,19 @@ def test_factory_lever_bad(test_url: str): ('https://youtube.com/watch?v=Gv8Wz74FjVA', 'youtube.com/watch'), ('https://i.imgur.com/BuzvZwb.gifv', 'i.imgur.com/BuzvZwb.gifv'), )) -def test_sanitise_urll(test_url: str, expected: str): - result = DownloadFactory._sanitise_url(test_url) +def test_sanitise_url(test_url: str, expected: str): + result = DownloadFactory.sanitise_url(test_url) + assert result == expected + + +@pytest.mark.parametrize(('test_url', 'expected'), ( + ('www.example.com/test.asp', True), + ('www.example.com/test.html', True), + ('www.example.com/test.js', True), + ('www.example.com/test.xhtml', True), + ('www.example.com/test.mp4', False), + ('www.example.com/test.png', False), +)) +def test_is_web_resource(test_url: str, expected: bool): + result = DownloadFactory.is_web_resource(test_url) assert result == expected