ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix download retries logic

Browse Source
This commit is contained in:
Serene-Arc
2021-05-25 09:56:22 +10:00
parent 4c42469c0c
commit 323b2d2b03
1 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
bdfr/resource.py
View File

@@ -28,8 +28,7 @@ class Resource:
self.extension = self._determine_extension() self.extension = self._determine_extension()
@staticmethod @staticmethod
def retry_download(url: str, max_wait_time: int) -> Optional[bytes]: def retry_download(url: str, max_wait_time: int, current_wait_time: int = 60) -> Optional[bytes]:
wait_time = 60
try: try:
response = requests.get(url) response = requests.get(url)
if re.match(r'^2\d{2}', str(response.status_code)) and response.content: if re.match(r'^2\d{2}', str(response.status_code)) and response.content:
@@ -40,10 +39,11 @@ class Resource:
raise BulkDownloaderException( raise BulkDownloaderException(
f'Unrecoverable error requesting resource: HTTP Code {response.status_code}') f'Unrecoverable error requesting resource: HTTP Code {response.status_code}')
except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError) as e: except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError) as e:
logger.warning(f'Error occured downloading from {url}, waiting {wait_time} seconds: {e}') logger.warning(f'Error occured downloading from {url}, waiting {current_wait_time} seconds: {e}')
time.sleep(wait_time) time.sleep(current_wait_time)
if wait_time < max_wait_time: if current_wait_time < max_wait_time:
return Resource.retry_download(url, max_wait_time) current_wait_time += 60
return Resource.retry_download(url, max_wait_time, current_wait_time)
else: else:
logger.error(f'Max wait time exceeded for resource at url {url}') logger.error(f'Max wait time exceeded for resource at url {url}')
raise raise