Fixed issue where file extenions not found and auth timeout
This commit is contained in:
+109
-11
@@ -23,6 +23,8 @@ from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import prawcore
|
||||
|
||||
from bdfr.configuration import Configuration
|
||||
from bdfr.connector import RedditConnector
|
||||
from bdfr.downloader import RedditDownloader
|
||||
@@ -33,6 +35,62 @@ from bdfr import exceptions as errors
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def retry_reddit_api_call(api_call_func: Callable, max_retries: int = 5, base_wait_time: int = 60) -> Any:
|
||||
"""
|
||||
Retry wrapper for Reddit API calls that handles rate limiting (429 errors).
|
||||
|
||||
Args:
|
||||
api_call_func: Function that makes the Reddit API call
|
||||
max_retries: Maximum number of retry attempts (default: 5)
|
||||
base_wait_time: Base wait time in seconds (default: 60)
|
||||
|
||||
Returns:
|
||||
Result of the API call
|
||||
|
||||
Raises:
|
||||
Exception: If all retry attempts are exhausted
|
||||
"""
|
||||
current_wait_time = base_wait_time
|
||||
max_wait_time = base_wait_time * max_retries # Total max wait time
|
||||
|
||||
for attempt in range(max_retries + 1): # +1 for initial attempt
|
||||
try:
|
||||
logger.debug(f"Reddit API call attempt {attempt + 1}/{max_retries + 1}")
|
||||
return api_call_func()
|
||||
|
||||
except Exception as e:
|
||||
# Check if this is a rate limiting error
|
||||
is_rate_limited = False
|
||||
|
||||
# Check for PRAW TooManyRequests exception
|
||||
if isinstance(e, prawcore.exceptions.TooManyRequests):
|
||||
is_rate_limited = True
|
||||
logger.warning(f"Reddit API rate limited (TooManyRequests): {e}")
|
||||
# Check for HTTP 429 in error message
|
||||
elif "429" in str(e):
|
||||
is_rate_limited = True
|
||||
logger.warning(f"Reddit API rate limited (HTTP 429): {e}")
|
||||
|
||||
if not is_rate_limited:
|
||||
# Not a rate limiting error, re-raise immediately
|
||||
logger.error(f"Reddit API call failed with non-rate-limit error: {e}")
|
||||
raise
|
||||
|
||||
# This is a rate limiting error
|
||||
if attempt == max_retries:
|
||||
# Last attempt failed, give up
|
||||
logger.error(f"Reddit API rate limit retry attempts exhausted ({max_retries + 1} attempts)")
|
||||
logger.error(f"Final error: {e}")
|
||||
raise Exception(f"Reddit API rate limited after {max_retries + 1} attempts: {e}")
|
||||
|
||||
# Wait before retrying
|
||||
logger.info(f"Reddit API rate limited, waiting {current_wait_time} seconds before retry {attempt + 1}/{max_retries}")
|
||||
time.sleep(current_wait_time)
|
||||
|
||||
# Increase wait time for next attempt (exponential backoff)
|
||||
current_wait_time = min(current_wait_time + base_wait_time, max_wait_time)
|
||||
|
||||
|
||||
class DownloadType(Enum):
|
||||
"""Types of downloads supported by BDFR"""
|
||||
SUBREDDIT = "subreddit"
|
||||
@@ -424,10 +482,15 @@ class BDFRManager:
|
||||
TooManyRequests = getattr(prawcore.exceptions, "TooManyRequests", None)
|
||||
if TooManyRequests is not None and isinstance(e, TooManyRequests):
|
||||
is_rate_limited = True
|
||||
self.logger.info(f"[DEBUG] Exception is TooManyRequests for download {download_id}")
|
||||
except Exception:
|
||||
pass
|
||||
if "429" in str(e):
|
||||
is_rate_limited = True
|
||||
self.logger.info(f"[DEBUG] Error message contains '429' for download {download_id}: {str(e)}")
|
||||
|
||||
self.logger.error(f"[DEBUG] Exception caught in _run_download for {download_id}: {e}")
|
||||
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
|
||||
|
||||
# Mark as failed
|
||||
download_info["status"] = DownloadStatus.FAILED.value
|
||||
@@ -556,10 +619,21 @@ class BDFRManager:
|
||||
)
|
||||
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
|
||||
|
||||
# Process this submission
|
||||
# Process this submission with retry mechanism for rate limiting
|
||||
self.logger.info(f"[DEBUG] Calling _download_submission for {submission.id}")
|
||||
downloader._download_submission(submission)
|
||||
self.logger.info(f"[DEBUG] Completed _download_submission for {submission.id}")
|
||||
try:
|
||||
downloader._download_submission(submission)
|
||||
self.logger.info(f"[DEBUG] Completed _download_submission for {submission.id}")
|
||||
except Exception as e:
|
||||
# Check if this is a rate limiting error that should be retried
|
||||
error_msg = str(e)
|
||||
if "429" in error_msg:
|
||||
self.logger.warning(f"[DEBUG] Rate limited while processing submission {submission.id}: {e}")
|
||||
self.logger.warning(f"[DEBUG] Will continue with next submission instead of failing entire download")
|
||||
continue
|
||||
else:
|
||||
# Not a rate limiting error, re-raise
|
||||
raise
|
||||
|
||||
# Update processed count
|
||||
download_info["items_processed"] = processed_submissions
|
||||
@@ -688,10 +762,21 @@ class BDFRManager:
|
||||
)
|
||||
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
|
||||
|
||||
# Archive this item
|
||||
# Archive this item with retry mechanism for rate limiting
|
||||
self.logger.info(f"[DEBUG] Calling write_entry for {item_id}")
|
||||
downloader.write_entry(item)
|
||||
self.logger.info(f"[DEBUG] Completed write_entry for {item_id}")
|
||||
try:
|
||||
downloader.write_entry(item)
|
||||
self.logger.info(f"[DEBUG] Completed write_entry for {item_id}")
|
||||
except Exception as e:
|
||||
# Check if this is a rate limiting error that should be retried
|
||||
error_msg = str(e)
|
||||
if "429" in error_msg:
|
||||
self.logger.warning(f"[DEBUG] Rate limited while archiving item {item_id}: {e}")
|
||||
self.logger.warning(f"[DEBUG] Will continue with next item instead of failing entire download")
|
||||
continue
|
||||
else:
|
||||
# Not a rate limiting error, re-raise
|
||||
raise
|
||||
|
||||
# Update processed count
|
||||
download_info["items_processed"] = processed_items
|
||||
@@ -801,12 +886,23 @@ class BDFRManager:
|
||||
)
|
||||
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
|
||||
|
||||
# Clone this submission (download + archive)
|
||||
# Clone this submission (download + archive) with retry mechanism for rate limiting
|
||||
self.logger.info(f"[DEBUG] Calling _download_submission for {submission.id}")
|
||||
downloader._download_submission(submission)
|
||||
self.logger.info(f"[DEBUG] Calling write_entry for {submission.id}")
|
||||
downloader.write_entry(submission)
|
||||
self.logger.info(f"[DEBUG] Completed cloning for {submission.id}")
|
||||
try:
|
||||
downloader._download_submission(submission)
|
||||
self.logger.info(f"[DEBUG] Calling write_entry for {submission.id}")
|
||||
downloader.write_entry(submission)
|
||||
self.logger.info(f"[DEBUG] Completed cloning for {submission.id}")
|
||||
except Exception as e:
|
||||
# Check if this is a rate limiting error that should be retried
|
||||
error_msg = str(e)
|
||||
if "429" in error_msg:
|
||||
self.logger.warning(f"[DEBUG] Rate limited while cloning submission {submission.id}: {e}")
|
||||
self.logger.warning(f"[DEBUG] Will continue with next submission instead of failing entire download")
|
||||
continue
|
||||
else:
|
||||
# Not a rate limiting error, re-raise
|
||||
raise
|
||||
|
||||
# Update processed count
|
||||
download_info["items_processed"] = processed_items
|
||||
@@ -820,6 +916,7 @@ class BDFRManager:
|
||||
if "429" in error_msg:
|
||||
self.logger.error(f"[DEBUG] Rate limited while cloning submission {submission_id}: {e}")
|
||||
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
|
||||
self.logger.info(f"[DEBUG] Error message contains '429': {error_msg}")
|
||||
download_info["status"] = DownloadStatus.FAILED.value
|
||||
download_info["error"] = f"Rate limited by Reddit API: {e}"
|
||||
download_info["end_time"] = datetime.now()
|
||||
@@ -870,6 +967,7 @@ class BDFRManager:
|
||||
stack_trace = traceback.format_exc()
|
||||
self.logger.error(f"Error in progress download for {download_id}: {e}")
|
||||
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
|
||||
self.logger.info(f"[DEBUG] Exception caught in progress_download, raising for {download_id}")
|
||||
raise
|
||||
|
||||
# Set up authentication if token provided
|
||||
|
||||
Reference in New Issue
Block a user