ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use config value to specify max wait time

Browse Source
This commit is contained in:
Serene-Arc
2021-04-12 16:47:04 +10:00
committed by Ali Parlakci
parent 0c960a4d0c
commit aa8032e95f
13 changed files with 19 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
bulkredditdownloader/resource.py
View File

@@ -27,7 +27,8 @@ class Resource:
self.extension = self._determine_extension()
@staticmethod
def retry_download(url: str, wait_time: int) -> Optional[bytes]:
def retry_download(url: str, max_wait_time: int) -> Optional[bytes]:
wait_time = 60
try:
response = requests.get(url)
if response.status_code == 200:
@@ -40,16 +41,16 @@ class Resource:
except requests.exceptions.ConnectionError as e:
logger.warning(f'Error occured downloading from {url}, waiting {wait_time} seconds: {e}')
time.sleep(wait_time)
if wait_time < 300:
return Resource.retry_download(url, wait_time + 60)
if wait_time < max_wait_time:
return Resource.retry_download(url, max_wait_time)
else:
logger.error(f'Max wait time exceeded for resource at url {url}')
raise
def download(self):
def download(self, max_wait_time: int):
if not self.content:
try:
content = self.retry_download(self.url, 0)
content = self.retry_download(self.url, max_wait_time)
except requests.exceptions.ConnectionError as e:
raise BulkDownloaderException(f'Could not download resource: {e}')
except BulkDownloaderException: