diff --git a/bulkredditdownloader/default_config.cfg b/bulkredditdownloader/default_config.cfg index 97d6bb9..1bcb02b 100644 --- a/bulkredditdownloader/default_config.cfg +++ b/bulkredditdownloader/default_config.cfg @@ -2,4 +2,5 @@ client_id = U-6gk4ZCh3IeNQ client_secret = 7CZHY6AmKweZME5s50SfDGylaPg scopes = identity, history, read, save -backup_log_count = 3 \ No newline at end of file +backup_log_count = 3 +max_wait_time = 120 \ No newline at end of file diff --git a/bulkredditdownloader/downloader.py b/bulkredditdownloader/downloader.py index 81a06c1..02c3511 100644 --- a/bulkredditdownloader/downloader.py +++ b/bulkredditdownloader/downloader.py @@ -362,7 +362,7 @@ class RedditDownloader: logger.debug(f'File {destination} already exists, continuing') else: try: - res.download() + res.download(self.cfg_parser.getint('DEFAULT', 'max_wait_time', fallback=120)) except errors.BulkDownloaderException as e: logger.error( f'Failed to download resource {res.url} with downloader {downloader_class.__name__}: {e}') diff --git a/bulkredditdownloader/resource.py b/bulkredditdownloader/resource.py index 92d1633..b925bd3 100644 --- a/bulkredditdownloader/resource.py +++ b/bulkredditdownloader/resource.py @@ -27,7 +27,8 @@ class Resource: self.extension = self._determine_extension() @staticmethod - def retry_download(url: str, wait_time: int) -> Optional[bytes]: + def retry_download(url: str, max_wait_time: int) -> Optional[bytes]: + wait_time = 60 try: response = requests.get(url) if response.status_code == 200: @@ -40,16 +41,16 @@ class Resource: except requests.exceptions.ConnectionError as e: logger.warning(f'Error occured downloading from {url}, waiting {wait_time} seconds: {e}') time.sleep(wait_time) - if wait_time < 300: - return Resource.retry_download(url, wait_time + 60) + if wait_time < max_wait_time: + return Resource.retry_download(url, max_wait_time) else: logger.error(f'Max wait time exceeded for resource at url {url}') raise - def download(self): + def download(self, max_wait_time: int): if not self.content: try: - content = self.retry_download(self.url, 0) + content = self.retry_download(self.url, max_wait_time) except requests.exceptions.ConnectionError as e: raise BulkDownloaderException(f'Could not download resource: {e}') except BulkDownloaderException: diff --git a/bulkredditdownloader/tests/site_downloaders/test_direct.py b/bulkredditdownloader/tests/site_downloaders/test_direct.py index 9f8163a..3e891b9 100644 --- a/bulkredditdownloader/tests/site_downloaders/test_direct.py +++ b/bulkredditdownloader/tests/site_downloaders/test_direct.py @@ -21,5 +21,5 @@ def test_download_resource(test_url: str, expected_hash: str): resources = test_site.find_resources() assert len(resources) == 1 assert isinstance(resources[0], Resource) - resources[0].download() + resources[0].download(120) assert resources[0].hash.hexdigest() == expected_hash diff --git a/bulkredditdownloader/tests/site_downloaders/test_erome.py b/bulkredditdownloader/tests/site_downloaders/test_erome.py index 2fb7cf6..8a052a0 100644 --- a/bulkredditdownloader/tests/site_downloaders/test_erome.py +++ b/bulkredditdownloader/tests/site_downloaders/test_erome.py @@ -52,6 +52,6 @@ def test_download_resource(test_url: str, expected_hashes: tuple[str]): mock_submission.url = test_url test_site = Erome(mock_submission) resources = test_site.find_resources() - [res.download() for res in resources] + [res.download(120) for res in resources] resource_hashes = [res.hash.hexdigest() for res in resources] assert len(resource_hashes) == len(expected_hashes) diff --git a/bulkredditdownloader/tests/site_downloaders/test_gallery.py b/bulkredditdownloader/tests/site_downloaders/test_gallery.py index d33c632..f61e04f 100644 --- a/bulkredditdownloader/tests/site_downloaders/test_gallery.py +++ b/bulkredditdownloader/tests/site_downloaders/test_gallery.py @@ -55,6 +55,6 @@ def test_gallery_download(test_submission_id: str, expected_hashes: set[str], re test_submission = reddit_instance.submission(id=test_submission_id) gallery = Gallery(test_submission) results = gallery.find_resources() - [res.download() for res in results] + [res.download(120) for res in results] hashes = [res.hash.hexdigest() for res in results] assert set(hashes) == expected_hashes diff --git a/bulkredditdownloader/tests/site_downloaders/test_gfycat.py b/bulkredditdownloader/tests/site_downloaders/test_gfycat.py index 5babe8c..8290914 100644 --- a/bulkredditdownloader/tests/site_downloaders/test_gfycat.py +++ b/bulkredditdownloader/tests/site_downloaders/test_gfycat.py @@ -32,5 +32,5 @@ def test_download_resource(test_url: str, expected_hash: str): resources = test_site.find_resources() assert len(resources) == 1 assert isinstance(resources[0], Resource) - resources[0].download() + resources[0].download(120) assert resources[0].hash.hexdigest() == expected_hash diff --git a/bulkredditdownloader/tests/site_downloaders/test_gif_delivery_network.py b/bulkredditdownloader/tests/site_downloaders/test_gif_delivery_network.py index a4399d8..9856536 100644 --- a/bulkredditdownloader/tests/site_downloaders/test_gif_delivery_network.py +++ b/bulkredditdownloader/tests/site_downloaders/test_gif_delivery_network.py @@ -33,5 +33,5 @@ def test_download_resource(test_url: str, expected_hash: str): resources = test_site.find_resources() assert len(resources) == 1 assert isinstance(resources[0], Resource) - resources[0].download() + resources[0].download(120) assert resources[0].hash.hexdigest() == expected_hash diff --git a/bulkredditdownloader/tests/site_downloaders/test_imgur.py b/bulkredditdownloader/tests/site_downloaders/test_imgur.py index bf842ff..2f22c2b 100644 --- a/bulkredditdownloader/tests/site_downloaders/test_imgur.py +++ b/bulkredditdownloader/tests/site_downloaders/test_imgur.py @@ -129,7 +129,7 @@ def test_find_resources(test_url: str, expected_hashes: list[str]): downloader = Imgur(mock_download) results = downloader.find_resources() assert all([isinstance(res, Resource) for res in results]) - [res.download() for res in results] + [res.download(120) for res in results] hashes = set([res.hash.hexdigest() for res in results]) assert len(results) == len(expected_hashes) assert hashes == set(expected_hashes) diff --git a/bulkredditdownloader/tests/site_downloaders/test_redgifs.py b/bulkredditdownloader/tests/site_downloaders/test_redgifs.py index 2aa4227..0d8cb79 100644 --- a/bulkredditdownloader/tests/site_downloaders/test_redgifs.py +++ b/bulkredditdownloader/tests/site_downloaders/test_redgifs.py @@ -33,5 +33,5 @@ def test_download_resource(test_url: str, expected_hash: str): resources = test_site.find_resources() assert len(resources) == 1 assert isinstance(resources[0], Resource) - resources[0].download() + resources[0].download(120) assert resources[0].hash.hexdigest() == expected_hash diff --git a/bulkredditdownloader/tests/site_downloaders/test_vreddit.py b/bulkredditdownloader/tests/site_downloaders/test_vreddit.py index bf96d67..d7377b6 100644 --- a/bulkredditdownloader/tests/site_downloaders/test_vreddit.py +++ b/bulkredditdownloader/tests/site_downloaders/test_vreddit.py @@ -19,5 +19,5 @@ def test_find_resources(test_submission_id: str, expected_hash: str, reddit_inst resources = downloader.find_resources() assert len(resources) == 1 assert isinstance(resources[0], Resource) - resources[0].download() + resources[0].download(120) assert resources[0].hash.hexdigest() == expected_hash diff --git a/bulkredditdownloader/tests/site_downloaders/test_youtube.py b/bulkredditdownloader/tests/site_downloaders/test_youtube.py index 8af08d0..8c7bfca 100644 --- a/bulkredditdownloader/tests/site_downloaders/test_youtube.py +++ b/bulkredditdownloader/tests/site_downloaders/test_youtube.py @@ -22,5 +22,5 @@ def test_find_resources(test_url: str, expected_hash: str): resources = downloader.find_resources() assert len(resources) == 1 assert isinstance(resources[0], Resource) - resources[0].download() + resources[0].download(120) assert resources[0].hash.hexdigest() == expected_hash diff --git a/bulkredditdownloader/tests/test_resource.py b/bulkredditdownloader/tests/test_resource.py index c2647b3..0292f06 100644 --- a/bulkredditdownloader/tests/test_resource.py +++ b/bulkredditdownloader/tests/test_resource.py @@ -28,5 +28,5 @@ def test_resource_get_extension(test_url: str, expected: str): )) def test_download_online_resource(test_url: str, expected_hash: str): test_resource = Resource(MagicMock(), test_url) - test_resource.download() + test_resource.download(120) assert test_resource.hash.hexdigest() == expected_hash