Fixed issue where file extenions not found and auth timeout

bdfr/api.py

@@ -23,6 +23,8 @@ from enum import Enum
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 
+import prawcore
+
 from bdfr.configuration import Configuration
 from bdfr.connector import RedditConnector
 from bdfr.downloader import RedditDownloader
@@ -33,6 +35,62 @@ from bdfr import exceptions as errors
 logger = logging.getLogger(__name__)
 
 
+def retry_reddit_api_call(api_call_func: Callable, max_retries: int = 5, base_wait_time: int = 60) -> Any:
+    """
+    Retry wrapper for Reddit API calls that handles rate limiting (429 errors).
+
+    Args:
+        api_call_func: Function that makes the Reddit API call
+        max_retries: Maximum number of retry attempts (default: 5)
+        base_wait_time: Base wait time in seconds (default: 60)
+
+    Returns:
+        Result of the API call
+
+    Raises:
+        Exception: If all retry attempts are exhausted
+    """
+    current_wait_time = base_wait_time
+    max_wait_time = base_wait_time * max_retries  # Total max wait time
+
+    for attempt in range(max_retries + 1):  # +1 for initial attempt
+        try:
+            logger.debug(f"Reddit API call attempt {attempt + 1}/{max_retries + 1}")
+            return api_call_func()
+
+        except Exception as e:
+            # Check if this is a rate limiting error
+            is_rate_limited = False
+
+            # Check for PRAW TooManyRequests exception
+            if isinstance(e, prawcore.exceptions.TooManyRequests):
+                is_rate_limited = True
+                logger.warning(f"Reddit API rate limited (TooManyRequests): {e}")
+            # Check for HTTP 429 in error message
+            elif "429" in str(e):
+                is_rate_limited = True
+                logger.warning(f"Reddit API rate limited (HTTP 429): {e}")
+
+            if not is_rate_limited:
+                # Not a rate limiting error, re-raise immediately
+                logger.error(f"Reddit API call failed with non-rate-limit error: {e}")
+                raise
+
+            # This is a rate limiting error
+            if attempt == max_retries:
+                # Last attempt failed, give up
+                logger.error(f"Reddit API rate limit retry attempts exhausted ({max_retries + 1} attempts)")
+                logger.error(f"Final error: {e}")
+                raise Exception(f"Reddit API rate limited after {max_retries + 1} attempts: {e}")
+
+            # Wait before retrying
+            logger.info(f"Reddit API rate limited, waiting {current_wait_time} seconds before retry {attempt + 1}/{max_retries}")
+            time.sleep(current_wait_time)
+
+            # Increase wait time for next attempt (exponential backoff)
+            current_wait_time = min(current_wait_time + base_wait_time, max_wait_time)
+
+
 class DownloadType(Enum):
     """Types of downloads supported by BDFR"""
     SUBREDDIT = "subreddit"
@@ -424,10 +482,15 @@ class BDFRManager:
                 TooManyRequests = getattr(prawcore.exceptions, "TooManyRequests", None)
                 if TooManyRequests is not None and isinstance(e, TooManyRequests):
                     is_rate_limited = True
+                    self.logger.info(f"[DEBUG] Exception is TooManyRequests for download {download_id}")
             except Exception:
                 pass
             if "429" in str(e):
                 is_rate_limited = True
+                self.logger.info(f"[DEBUG] Error message contains '429' for download {download_id}: {str(e)}")
+
+            self.logger.error(f"[DEBUG] Exception caught in _run_download for {download_id}: {e}")
+            self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
 
             # Mark as failed
             download_info["status"] = DownloadStatus.FAILED.value
@@ -556,10 +619,21 @@ class BDFRManager:
                                 )
                                 asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
 
-                                # Process this submission
+                                # Process this submission with retry mechanism for rate limiting
                                 self.logger.info(f"[DEBUG] Calling _download_submission for {submission.id}")
-                                downloader._download_submission(submission)
-                                self.logger.info(f"[DEBUG] Completed _download_submission for {submission.id}")
+                                try:
+                                    downloader._download_submission(submission)
+                                    self.logger.info(f"[DEBUG] Completed _download_submission for {submission.id}")
+                                except Exception as e:
+                                    # Check if this is a rate limiting error that should be retried
+                                    error_msg = str(e)
+                                    if "429" in error_msg:
+                                        self.logger.warning(f"[DEBUG] Rate limited while processing submission {submission.id}: {e}")
+                                        self.logger.warning(f"[DEBUG] Will continue with next submission instead of failing entire download")
+                                        continue
+                                    else:
+                                        # Not a rate limiting error, re-raise
+                                        raise
 
                                 # Update processed count
                                 download_info["items_processed"] = processed_submissions
@@ -688,10 +762,21 @@ class BDFRManager:
                                 )
                                 asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
                                 
-                                # Archive this item
+                                # Archive this item with retry mechanism for rate limiting
                                 self.logger.info(f"[DEBUG] Calling write_entry for {item_id}")
-                                downloader.write_entry(item)
-                                self.logger.info(f"[DEBUG] Completed write_entry for {item_id}")
+                                try:
+                                    downloader.write_entry(item)
+                                    self.logger.info(f"[DEBUG] Completed write_entry for {item_id}")
+                                except Exception as e:
+                                    # Check if this is a rate limiting error that should be retried
+                                    error_msg = str(e)
+                                    if "429" in error_msg:
+                                        self.logger.warning(f"[DEBUG] Rate limited while archiving item {item_id}: {e}")
+                                        self.logger.warning(f"[DEBUG] Will continue with next item instead of failing entire download")
+                                        continue
+                                    else:
+                                        # Not a rate limiting error, re-raise
+                                        raise
                                 
                                 # Update processed count
                                 download_info["items_processed"] = processed_items
@@ -801,12 +886,23 @@ class BDFRManager:
                                 )
                                 asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
                                 
-                                # Clone this submission (download + archive)
+                                # Clone this submission (download + archive) with retry mechanism for rate limiting
                                 self.logger.info(f"[DEBUG] Calling _download_submission for {submission.id}")
-                                downloader._download_submission(submission)
-                                self.logger.info(f"[DEBUG] Calling write_entry for {submission.id}")
-                                downloader.write_entry(submission)
-                                self.logger.info(f"[DEBUG] Completed cloning for {submission.id}")
+                                try:
+                                    downloader._download_submission(submission)
+                                    self.logger.info(f"[DEBUG] Calling write_entry for {submission.id}")
+                                    downloader.write_entry(submission)
+                                    self.logger.info(f"[DEBUG] Completed cloning for {submission.id}")
+                                except Exception as e:
+                                    # Check if this is a rate limiting error that should be retried
+                                    error_msg = str(e)
+                                    if "429" in error_msg:
+                                        self.logger.warning(f"[DEBUG] Rate limited while cloning submission {submission.id}: {e}")
+                                        self.logger.warning(f"[DEBUG] Will continue with next submission instead of failing entire download")
+                                        continue
+                                    else:
+                                        # Not a rate limiting error, re-raise
+                                        raise
                                 
                                 # Update processed count
                                 download_info["items_processed"] = processed_items
@@ -820,6 +916,7 @@ class BDFRManager:
                                 if "429" in error_msg:
                                     self.logger.error(f"[DEBUG] Rate limited while cloning submission {submission_id}: {e}")
                                     self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
+                                    self.logger.info(f"[DEBUG] Error message contains '429': {error_msg}")
                                     download_info["status"] = DownloadStatus.FAILED.value
                                     download_info["error"] = f"Rate limited by Reddit API: {e}"
                                     download_info["end_time"] = datetime.now()
@@ -870,6 +967,7 @@ class BDFRManager:
                 stack_trace = traceback.format_exc()
                 self.logger.error(f"Error in progress download for {download_id}: {e}")
                 self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
+                self.logger.info(f"[DEBUG] Exception caught in progress_download, raising for {download_id}")
                 raise
 
         # Set up authentication if token provided

bdfr/connector.py

@@ -413,8 +413,10 @@ class RedditConnector(metaclass=ABCMeta):
                     is_rate_limited = False
                     if TooManyRequests is not None and isinstance(e, TooManyRequests):
                         is_rate_limited = True
+                        logger.info(f"Rate limited detected: Exception is TooManyRequests for user {user}")
                     elif (hasattr(e, "response") and getattr(e.response, "status_code", None) == 429) or "429" in str(e):
                         is_rate_limited = True
+                        logger.info(f"Rate limited detected: Status code 429 or '429' in error message for user {user}. Error: {e}")
 
                     if is_rate_limited:
                         logger.error("Received HTTP 429 (rate limited). Propagating error to fail the download.")

bdfr/downloader.py

@@ -149,7 +149,9 @@ class RedditDownloader(RedditConnector):
             logger.error(f"Site {downloader_class.__name__} failed to download submission {submission.id}: {e}")
             return
         files_processed = 0
+        logger.debug(f"Processing {len(content)} resources for submission {submission.id}")
         for destination, res in self.file_name_formatter.format_resource_paths(content, self.download_directory):
+            logger.debug(f"Resource URL: {res.url}, Extension: {res.extension}, Destination: {destination}")
             if destination.exists():
                 # Check if we already have this file's hash
                 if destination in self.master_hash_list.values():
@@ -217,9 +219,10 @@ class RedditDownloader(RedditConnector):
             # Only create folder if we're actually going to write the file (not a duplicate)
             destination.parent.mkdir(parents=True, exist_ok=True)
             try:
+                logger.debug(f"Writing {len(res.content)} bytes to {destination}")
                 with destination.open("wb") as file:
                     file.write(res.content)
-                logger.debug(f"Written file to {destination}")
+                logger.debug(f"Successfully written file to {destination}")
                 files_processed += 1
             except OSError as e:
                 logger.exception(e)

bdfr/file_name_formatter.py

@@ -126,6 +126,7 @@ class FileNameFormatter:
         )
         index = f"_{index}" if index else ""
         if not resource.extension:
+            logger.error(f"Resource from {resource.url} has no extension - URL: {resource.url}")
             raise BulkDownloaderException(f"Resource from {resource.url} has no extension")
         file_name = str(self._format_name(resource.source_submission, self.file_format_string))
 

bdfr/resource.py

@@ -24,7 +24,7 @@ class Resource:
         self.content: Optional[bytes] = None
         self.url = url
         self.hash: Optional[_hashlib.HASH] = None
-        self.extension = extension
+        self.extension = self._normalize_extension(extension)
         self.download_function = download_function
         if not self.extension:
             self.extension = self._determine_extension()
@@ -45,6 +45,13 @@ class Resource:
                 raise
             if content:
                 self.content = content
+
+                # If we didn't have an extension before, try to detect from content
+                if not self.extension:
+                    logger.debug(f"Attempting content-based extension detection for {self.url}")
+                    detected = self._detect_extension_by_content()
+                    self.extension = self._normalize_extension(detected) if detected else None
+
         if not self.hash and self.content:
             self.create_hash()
 
@@ -54,9 +61,69 @@ class Resource:
     def _determine_extension(self) -> Optional[str]:
         extension_pattern = re.compile(r".*(\..{3,5})$")
         stripped_url = urllib.parse.urlsplit(self.url).path
+
+        # Special handling for Reddit media URLs
+        if self.url.startswith("https://www.reddit.com/media"):
+            logger.debug(f"Detected Reddit media URL: {self.url}")
+            parsed_url = urllib.parse.urlparse(self.url)
+            url_param = urllib.parse.parse_qs(parsed_url.query).get('url', [None])[0]
+            if url_param:
+                decoded_url = urllib.parse.unquote(url_param)
+                logger.debug(f"Reddit media URL decoded to: {decoded_url}")
+                stripped_url = urllib.parse.urlsplit(decoded_url).path
+
+        # Also handle preview.redd.it URLs which might not have extensions
+        elif "preview.redd.it" in self.url and not stripped_url.endswith(('.jpg', '.jpeg', '.png', '.gif', '.webp')):
+            logger.debug(f"Detected preview.redd.it URL without extension: {self.url}")
+            # For preview URLs, try to infer from common patterns or add fallback logic
+
         match = re.search(extension_pattern, stripped_url)
         if match:
-            return match.group(1)
+            extension = match.group(1)
+            logger.debug(f"URL {self.url} -> extracted extension: {extension} (from path: {stripped_url})")
+            return self._normalize_extension(extension)
+        else:
+            logger.warning(f"Could not determine extension for URL: {self.url} (path: {stripped_url})")
+
+            # As a last resort, if we have content, try to detect by magic numbers
+            if hasattr(self, 'content') and self.content:
+                detected = self._detect_extension_by_content()
+                return self._normalize_extension(detected) if detected else None
+
+            return None
+
+    def _detect_extension_by_content(self) -> Optional[str]:
+        """Detect file extension by examining file content (magic numbers)"""
+        if not self.content or len(self.content) < 16:
+            return None
+
+        # Check for common image formats
+        if self.content.startswith(b'\xFF\xD8\xFF'):
+            logger.debug(f"Detected JPEG by magic number for URL: {self.url}")
+            return '.jpg'
+        elif self.content.startswith(b'\x89PNG\r\n\x1a\n'):
+            logger.debug(f"Detected PNG by magic number for URL: {self.url}")
+            return '.png'
+        elif self.content.startswith(b'GIF87a') or self.content.startswith(b'GIF89a'):
+            logger.debug(f"Detected GIF by magic number for URL: {self.url}")
+            return '.gif'
+        elif self.content.startswith(b'RIFF') and self.content[8:12] == b'WEBP':
+            logger.debug(f"Detected WebP by magic number for URL: {self.url}")
+            return '.webp'
+        elif self.content.startswith(b'BM'):
+            logger.debug(f"Detected BMP by magic number for URL: {self.url}")
+            return '.bmp'
+
+        logger.debug(f"Could not detect file type by magic number for URL: {self.url}")
+        return None
+
+    def _normalize_extension(self, extension: Optional[str]) -> Optional[str]:
+        """Normalize extension to lowercase for consistency"""
+        if not extension:
+            return None
+        normalized = extension.lower()
+        logger.debug(f"Normalized extension '{extension}' to '{normalized}'")
+        return normalized
 
     @staticmethod
     def http_download(url: str, download_parameters: dict) -> Optional[bytes]:

bdfr/site_downloaders/download_factory.py

@@ -24,8 +24,14 @@ from bdfr.site_downloaders.youtube import Youtube
 class DownloadFactory:
     @staticmethod
     def pull_lever(url: str) -> type[BaseDownloader]:
+        import logging
+        logger = logging.getLogger(__name__)
+
         sanitised_url = DownloadFactory.sanitise_url(url).lower()
+        logger.debug(f"Selecting downloader for URL: {url} (sanitized: {sanitised_url})")
+
         if re.match(r"(i\.|m\.|o\.)?imgur", sanitised_url):
+            logger.debug("Using Imgur downloader")
             return Imgur
         elif re.match(r"(i\.|thumbs\d\.|v\d\.)?(redgifs|gifdeliverynetwork)", sanitised_url):
             return Redgifs

bdfr/site_downloaders/fallback_downloaders/ytdlp_fallback.py

@@ -20,12 +20,19 @@ class YtdlpFallback(BaseFallbackDownloader, Youtube):
         super(YtdlpFallback, self).__init__(post)
 
     def find_resources(self, authenticator: Optional[SiteAuthenticator] = None) -> list[Resource]:
+        logger.debug(f"YtdlpFallback processing URL: {self.post.url}")
+        video_attrs = super().get_video_attributes(self.post.url)
+        logger.debug(f"Video attributes: {video_attrs}")
+        extension = video_attrs.get("ext", None)
+        logger.debug(f"Using extension: {extension}")
+
         out = Resource(
             self.post,
             self.post.url,
             super()._download_video({}),
-            super().get_video_attributes(self.post.url)["ext"],
+            extension,
         )
+        logger.debug(f"Created resource with extension: {out.extension}")
         return [out]
 
     @staticmethod