diff --git a/bdfr/__main__.py b/bdfr/__main__.py index 4d12cad..ef90abe 100644 --- a/bdfr/__main__.py +++ b/bdfr/__main__.py @@ -26,7 +26,11 @@ _common_options = [ click.option("--file-scheme", default=None, type=str), click.option("--filename-restriction-scheme", type=click.Choice(("linux", "windows")), default=None), click.option("--folder-scheme", default=None, type=str), - click.option("--strip-unicode/--no-strip-unicode", default=None, help="Strip Unicode characters that cause Windows SMB issues (default: enabled)"), + click.option( + "--strip-unicode/--no-strip-unicode", + default=None, + help="Strip Unicode characters that cause Windows SMB issues (default: enabled)", + ), click.option("--ignore-user", type=str, multiple=True, default=None), click.option("--include-id-file", multiple=True, default=None), click.option("--log", type=str, default=None), @@ -54,7 +58,12 @@ _downloader_options = [ click.option("--max-wait-time", type=int, default=None), click.option("--no-dupes", is_flag=True, default=None), click.option("--search-existing", is_flag=True, default=None), - click.option("--simple-check", is_flag=True, default=None, help="Enable fast URL-based duplicate checking (works with --no-dupes)"), + click.option( + "--simple-check", + is_flag=True, + default=None, + help="Enable fast URL-based duplicate checking (works with --no-dupes)", + ), click.option("--skip", default=None, multiple=True), click.option("--skip-domain", default=None, multiple=True), click.option("--skip-subreddit", default=None, multiple=True), diff --git a/bdfr/api.py b/bdfr/api.py index f5dd528..019c3a5 100644 --- a/bdfr/api.py +++ b/bdfr/api.py @@ -25,12 +25,12 @@ from typing import Any, Dict, List, Optional, Union import prawcore +from bdfr import exceptions as errors +from bdfr.archiver import Archiver +from bdfr.cloner import RedditCloner from bdfr.configuration import Configuration from bdfr.connector import RedditConnector from bdfr.downloader import RedditDownloader -from bdfr.archiver import Archiver -from bdfr.cloner import RedditCloner -from bdfr import exceptions as errors logger = logging.getLogger(__name__) @@ -84,7 +84,9 @@ def retry_reddit_api_call(api_call_func: Callable, max_retries: int = 5, base_wa raise Exception(f"Reddit API rate limited after {max_retries + 1} attempts: {e}") # Wait before retrying - logger.info(f"Reddit API rate limited, waiting {current_wait_time} seconds before retry {attempt + 1}/{max_retries}") + logger.info( + f"Reddit API rate limited, waiting {current_wait_time} seconds before retry {attempt + 1}/{max_retries}" + ) time.sleep(current_wait_time) # Increase wait time for next attempt (exponential backoff) @@ -93,6 +95,7 @@ def retry_reddit_api_call(api_call_func: Callable, max_retries: int = 5, base_wa class DownloadType(Enum): """Types of downloads supported by BDFR""" + SUBREDDIT = "subreddit" USER = "user" MULTIREDDIT = "multireddit" @@ -103,6 +106,7 @@ class DownloadType(Enum): class DownloadStatus(Enum): """Status of a download operation""" + QUEUED = "queued" RUNNING = "running" COMPLETED = "completed" @@ -121,7 +125,7 @@ class ProgressEvent: message: str, progress: Optional[float] = None, data: Optional[Dict[str, Any]] = None, - timestamp: Optional[datetime] = None + timestamp: Optional[datetime] = None, ): self.event_type = event_type # "progress", "status", "error", "completed" self.download_id = download_id @@ -138,7 +142,7 @@ class ProgressEvent: "message": self.message, "progress": self.progress, "data": self.data, - "timestamp": self.timestamp.isoformat() + "timestamp": self.timestamp.isoformat(), } @@ -177,8 +181,8 @@ class LoggingCallback(ProgressCallback): async def on_error(self, event: ProgressEvent): """Log error events""" self.logger.error(f"[{event.download_id}] ERROR: {event.message}") - if event.data.get('exception'): - self.logger.exception(f"[{event.download_id}] Exception details", exc_info=event.data['exception']) + if event.data.get("exception"): + self.logger.exception(f"[{event.download_id}] Exception details", exc_info=event.data["exception"]) async def on_completed(self, event: ProgressEvent): """Log completion events""" @@ -258,7 +262,7 @@ class BDFRManager: download_type: DownloadType, name: str, config: Optional[Configuration] = None, - progress_callbacks: Optional[List[ProgressCallback]] = None + progress_callbacks: Optional[List[ProgressCallback]] = None, ) -> str: """ Create a new download operation. @@ -326,7 +330,7 @@ class BDFRManager: "items_processed": 0, "items_found": 0, "current_item": None, - "phase": "queued" + "phase": "queued", } self.logger.info(f"Created download {download_id} for {download_type.value}: {name}") @@ -356,11 +360,7 @@ class BDFRManager: download_info["start_time"] = datetime.now() # Create and start download thread - download_thread = threading.Thread( - target=self._run_download, - args=(download_id,), - daemon=True - ) + download_thread = threading.Thread(target=self._run_download, args=(download_id,), daemon=True) self._download_threads[download_id] = download_thread download_thread.start() @@ -375,8 +375,12 @@ class BDFRManager: downloader = None try: - self.logger.info(f"[DEBUG] Starting download {download_id} for {download_info['type']}: {download_info['name']}") - self.logger.info(f"[DEBUG] Config: subreddit={getattr(config, 'subreddit', None)}, user={getattr(config, 'user', None)}, limit={getattr(config, 'limit', None)}") + self.logger.info( + f"[DEBUG] Starting download {download_id} for {download_info['type']}: {download_info['name']}" + ) + self.logger.info( + f"[DEBUG] Config: subreddit={getattr(config, 'subreddit', None)}, user={getattr(config, 'user', None)}, limit={getattr(config, 'limit', None)}" + ) # Create progress event for start start_event = ProgressEvent( @@ -384,11 +388,13 @@ class BDFRManager: download_id, f"Starting {download_info['type']} download: {download_info['name']}", 0.0, - {"phase": "starting"} + {"phase": "starting"}, ) # Notify callbacks - self.logger.info(f"[BDFR-API] Notifying {len(callbacks)} callbacks of progress event for download {download_id}") + self.logger.info( + f"[BDFR-API] Notifying {len(callbacks)} callbacks of progress event for download {download_id}" + ) asyncio.run(self._notify_callbacks(callbacks, "on_progress", start_event)) # Set up authentication token in config if provided @@ -411,7 +417,9 @@ class BDFRManager: log_path.parent.mkdir(parents=True, exist_ok=True) # Determine which BDFR class to use based on configuration - self.logger.info(f"[DEBUG] Determining downloader type - submitted={getattr(config, 'submitted', False)}, upvoted={getattr(config, 'upvoted', False)}, saved={getattr(config, 'saved', False)}, format={getattr(config, 'format', 'N/A')}") + self.logger.info( + f"[DEBUG] Determining downloader type - submitted={getattr(config, 'submitted', False)}, upvoted={getattr(config, 'upvoted', False)}, saved={getattr(config, 'saved', False)}, format={getattr(config, 'format', 'N/A')}" + ) try: if config.submitted or config.upvoted or config.saved: @@ -420,7 +428,7 @@ class BDFRManager: downloader = RedditDownloader(config, []) download_info["operation_type"] = "user_download" self.logger.info(f"[DEBUG] Using RedditDownloader for user data") - elif hasattr(config, 'format') and config.format in ["json", "xml", "yaml"]: + elif hasattr(config, "format") and config.format in ["json", "xml", "yaml"]: # Archive operation self.logger.info(f"[DEBUG] Creating Archiver for archive operation") downloader = Archiver(config, []) @@ -447,7 +455,9 @@ class BDFRManager: # Mark as completed only if not already failed or cancelled if download_info.get("status") not in [DownloadStatus.FAILED.value, DownloadStatus.CANCELLED.value]: self.logger.info(f"[DEBUG] Marking download {download_id} as completed") - self.logger.info(f"[DEBUG] Final stats - items_processed: {download_info['items_processed']}, items_found: {download_info['items_found']}") + self.logger.info( + f"[DEBUG] Final stats - items_processed: {download_info['items_processed']}, items_found: {download_info['items_found']}" + ) download_info["status"] = DownloadStatus.COMPLETED.value download_info["progress"] = 100.0 download_info["end_time"] = datetime.now() @@ -460,18 +470,21 @@ class BDFRManager: { "items_processed": download_info["items_processed"], "items_found": download_info["items_found"], - "operation_type": download_info.get("operation_type") - } + "operation_type": download_info.get("operation_type"), + }, ) self.logger.info(f"[DEBUG] Sending completion callback for {download_id}") self.logger.info(f"[BDFR-API] Sending completion event for download {download_id}") asyncio.run(self._notify_callbacks(callbacks, "on_completed", completed_event)) else: - self.logger.info(f"[DEBUG] Skipping completion for {download_id} due to status={download_info.get('status')}") + self.logger.info( + f"[DEBUG] Skipping completion for {download_id} due to status={download_info.get('status')}" + ) except Exception as e: import traceback + # Get full stack trace for debugging stack_trace = traceback.format_exc() @@ -479,6 +492,7 @@ class BDFRManager: is_rate_limited = False try: import prawcore + TooManyRequests = getattr(prawcore.exceptions, "TooManyRequests", None) if TooManyRequests is not None and isinstance(e, TooManyRequests): is_rate_limited = True @@ -503,7 +517,11 @@ class BDFRManager: download_id, f"Download failed: {str(e)}", None, - {"exception": str(e), "stack_trace": stack_trace, "phase": ("rate_limited" if is_rate_limited else "failed")} + { + "exception": str(e), + "stack_trace": stack_trace, + "phase": ("rate_limited" if is_rate_limited else "failed"), + }, ) asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event)) @@ -519,17 +537,19 @@ class BDFRManager: # Close any logging handlers that might be holding file locks logger = logging.getLogger() for handler in logger.handlers[:]: - if hasattr(handler, 'close') and hasattr(handler, 'baseFilename'): + if hasattr(handler, "close") and hasattr(handler, "baseFilename"): # Check if this handler is for our download's log file - if hasattr(config, 'log') and config.log: - handler_path = getattr(handler, 'baseFilename', '') + if hasattr(config, "log") and config.log: + handler_path = getattr(handler, "baseFilename", "") if handler_path == config.log or handler_path.startswith(str(self.download_directory)): try: handler.close() logger.removeHandler(handler) self.logger.info(f"[DEBUG] Closed logging handler for {handler_path}") except Exception as cleanup_error: - self.logger.warning(f"[DEBUG] Error closing handler {handler_path}: {cleanup_error}") + self.logger.warning( + f"[DEBUG] Error closing handler {handler_path}: {cleanup_error}" + ) except Exception as cleanup_error: self.logger.warning(f"[DEBUG] Error during logging cleanup: {cleanup_error}") @@ -549,17 +569,13 @@ class BDFRManager: # Update phase to fetching download_info["phase"] = "fetching_submissions" progress_event = ProgressEvent( - "progress", - download_id, - "Fetching submissions...", - 10.0, - {"phase": "fetching_submissions"} + "progress", download_id, "Fetching submissions...", 10.0, {"phase": "fetching_submissions"} ) self.logger.info(f"[BDFR-API] Sending progress update for download {download_id}") asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) # For RedditDownloader, we need to intercept the download process - if hasattr(downloader, 'reddit_lists') and hasattr(downloader, '_download_submission'): + if hasattr(downloader, "reddit_lists") and hasattr(downloader, "_download_submission"): processed_submissions = 0 self.logger.info(f"[DEBUG] RedditDownloader has reddit_lists and _download_submission methods") @@ -567,7 +583,7 @@ class BDFRManager: # Get the limit from config - this is our target/goal for items_found # items_found represents "how many items are available to download (up to limit)" # items_processed represents "how many we've actually processed so far" - limit = getattr(config, 'limit', 100) + limit = getattr(config, "limit", 100) download_info["items_found"] = limit self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}") @@ -590,7 +606,9 @@ class BDFRManager: return try: - self.logger.info(f"[DEBUG] Processing submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}") + self.logger.info( + f"[DEBUG] Processing submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}" + ) # Update current item being processed download_info["current_item"] = submission.id @@ -614,8 +632,8 @@ class BDFRManager: "current_item_type": "submission", "current_subreddit": submission.subreddit.display_name, "items_processed": processed_submissions, - "items_found": download_info["items_found"] # Keep showing the limit as target - } + "items_found": download_info["items_found"], # Keep showing the limit as target + }, ) asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) @@ -628,8 +646,12 @@ class BDFRManager: # Check if this is a rate limiting error that should be retried error_msg = str(e) if "429" in error_msg: - self.logger.warning(f"[DEBUG] Rate limited while processing submission {submission.id}: {e}") - self.logger.warning(f"[DEBUG] Will continue with next submission instead of failing entire download") + self.logger.warning( + f"[DEBUG] Rate limited while processing submission {submission.id}: {e}" + ) + self.logger.warning( + f"[DEBUG] Will continue with next submission instead of failing entire download" + ) continue else: # Not a rate limiting error, re-raise @@ -640,6 +662,7 @@ class BDFRManager: except Exception as e: import traceback + error_msg = str(e) submission_id = submission.id if submission is not None else "unknown" @@ -647,7 +670,9 @@ class BDFRManager: stack_trace = traceback.format_exc() if "429" in error_msg: - self.logger.error(f"[DEBUG] Rate limited while processing submission {submission_id}: {e}") + self.logger.error( + f"[DEBUG] Rate limited while processing submission {submission_id}: {e}" + ) self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") # Mark download as failed due to rate limiting download_info["status"] = DownloadStatus.FAILED.value @@ -659,7 +684,7 @@ class BDFRManager: download_id, f"Rate limited by Reddit API: {e}", None, - {"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"} + {"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"}, ) asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event)) return @@ -675,9 +700,11 @@ class BDFRManager: if actual_count < limit: download_info["items_found"] = actual_count self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}") - - self.logger.info(f"[DEBUG] Final progress update - processed {processed_submissions} out of {download_info['items_found']} found") - + + self.logger.info( + f"[DEBUG] Final progress update - processed {processed_submissions} out of {download_info['items_found']} found" + ) + progress_event = ProgressEvent( "progress", download_id, @@ -686,26 +713,26 @@ class BDFRManager: { "phase": "completed", "items_processed": processed_submissions, - "items_found": download_info["items_found"] - } + "items_found": download_info["items_found"], + }, ) asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) - elif hasattr(downloader, 'reddit_lists') and hasattr(downloader, 'write_entry'): + elif hasattr(downloader, "reddit_lists") and hasattr(downloader, "write_entry"): # For Archiver, we need to intercept the archiving process processed_items = 0 - + self.logger.info(f"[DEBUG] Archiver detected with reddit_lists and write_entry methods") - + # Get the limit from config - limit = getattr(config, 'limit', 100) + limit = getattr(config, "limit", 100) download_info["items_found"] = limit self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}") - + # Process items with progress tracking self.logger.info(f"[DEBUG] Starting archiving loop...") actual_count = 0 - + for generator in downloader.reddit_lists: self.logger.info(f"[DEBUG] Processing generator for archiving...") item_count = 0 @@ -714,14 +741,15 @@ class BDFRManager: continue item_count += 1 actual_count += 1 - + if download_info["status"] == DownloadStatus.CANCELLED.value: self.logger.info(f"Download {download_id} was cancelled") return - + try: # Determine item type import praw.models + if isinstance(item, praw.models.Submission): item_id = item.id item_type = "submission" @@ -729,23 +757,27 @@ class BDFRManager: elif isinstance(item, praw.models.Comment): item_id = item.id item_type = "comment" - subreddit_name = item.subreddit.display_name if hasattr(item, 'subreddit') else "unknown" + subreddit_name = ( + item.subreddit.display_name if hasattr(item, "subreddit") else "unknown" + ) else: item_id = str(item) item_type = "item" subreddit_name = "unknown" - - self.logger.info(f"[DEBUG] Archiving {item_type} {item_count}: {item_id} from r/{subreddit_name}") - + + self.logger.info( + f"[DEBUG] Archiving {item_type} {item_count}: {item_id} from r/{subreddit_name}" + ) + # Update current item being processed download_info["current_item"] = item_id download_info["current_item_type"] = item_type download_info["current_subreddit"] = subreddit_name - + # Send progress update for this item processed_items += 1 progress_percent = 20.0 + (processed_items / max(limit, 1)) * 70.0 - + progress_event = ProgressEvent( "progress", download_id, @@ -757,11 +789,11 @@ class BDFRManager: "current_item_type": item_type, "current_subreddit": subreddit_name, "items_processed": processed_items, - "items_found": download_info["items_found"] - } + "items_found": download_info["items_found"], + }, ) asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) - + # Archive this item with retry mechanism for rate limiting self.logger.info(f"[DEBUG] Calling write_entry for {item_id}") try: @@ -772,34 +804,37 @@ class BDFRManager: error_msg = str(e) if "429" in error_msg: self.logger.warning(f"[DEBUG] Rate limited while archiving item {item_id}: {e}") - self.logger.warning(f"[DEBUG] Will continue with next item instead of failing entire download") + self.logger.warning( + f"[DEBUG] Will continue with next item instead of failing entire download" + ) continue else: # Not a rate limiting error, re-raise raise - + # Update processed count download_info["items_processed"] = processed_items - + except Exception as e: import traceback + error_msg = str(e) - item_id = item_id if 'item_id' in locals() else "unknown" + item_id = item_id if "item_id" in locals() else "unknown" stack_trace = traceback.format_exc() - + if "429" in error_msg: self.logger.error(f"[DEBUG] Rate limited while archiving item {item_id}: {e}") self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") download_info["status"] = DownloadStatus.FAILED.value download_info["error"] = f"Rate limited by Reddit API: {e}" download_info["end_time"] = datetime.now() - + error_event = ProgressEvent( "error", download_id, f"Rate limited by Reddit API: {e}", None, - {"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"} + {"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"}, ) asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event)) return @@ -807,16 +842,18 @@ class BDFRManager: self.logger.error(f"[DEBUG] Error archiving item {item_id}: {e}") self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") continue - + self.logger.info(f"[DEBUG] Generator archived {item_count} items") - + # Final progress update if actual_count < limit: download_info["items_found"] = actual_count self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}") - - self.logger.info(f"[DEBUG] Final progress update - archived {processed_items} out of {download_info['items_found']} found") - + + self.logger.info( + f"[DEBUG] Final progress update - archived {processed_items} out of {download_info['items_found']} found" + ) + progress_event = ProgressEvent( "progress", download_id, @@ -825,26 +862,32 @@ class BDFRManager: { "phase": "completed", "items_processed": processed_items, - "items_found": download_info["items_found"] - } + "items_found": download_info["items_found"], + }, ) asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) - - elif hasattr(downloader, 'reddit_lists') and hasattr(downloader, '_download_submission') and hasattr(downloader, 'write_entry'): + + elif ( + hasattr(downloader, "reddit_lists") + and hasattr(downloader, "_download_submission") + and hasattr(downloader, "write_entry") + ): # For RedditCloner (which has both download and archive capabilities) processed_items = 0 - - self.logger.info(f"[DEBUG] RedditCloner detected with reddit_lists, _download_submission, and write_entry methods") - + + self.logger.info( + f"[DEBUG] RedditCloner detected with reddit_lists, _download_submission, and write_entry methods" + ) + # Get the limit from config - limit = getattr(config, 'limit', 100) + limit = getattr(config, "limit", 100) download_info["items_found"] = limit self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}") - + # Process submissions with progress tracking (both download and archive) self.logger.info(f"[DEBUG] Starting cloning loop...") actual_count = 0 - + for generator in downloader.reddit_lists: self.logger.info(f"[DEBUG] Processing generator for cloning...") submission_count = 0 @@ -853,23 +896,25 @@ class BDFRManager: continue submission_count += 1 actual_count += 1 - + if download_info["status"] == DownloadStatus.CANCELLED.value: self.logger.info(f"Download {download_id} was cancelled") return - + try: - self.logger.info(f"[DEBUG] Cloning submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}") - + self.logger.info( + f"[DEBUG] Cloning submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}" + ) + # Update current item being processed download_info["current_item"] = submission.id download_info["current_item_type"] = "submission" download_info["current_subreddit"] = submission.subreddit.display_name - + # Send progress update for this submission processed_items += 1 progress_percent = 20.0 + (processed_items / max(limit, 1)) * 70.0 - + progress_event = ProgressEvent( "progress", download_id, @@ -881,11 +926,11 @@ class BDFRManager: "current_item_type": "submission", "current_subreddit": submission.subreddit.display_name, "items_processed": processed_items, - "items_found": download_info["items_found"] - } + "items_found": download_info["items_found"], + }, ) asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) - + # Clone this submission (download + archive) with retry mechanism for rate limiting self.logger.info(f"[DEBUG] Calling _download_submission for {submission.id}") try: @@ -897,36 +942,43 @@ class BDFRManager: # Check if this is a rate limiting error that should be retried error_msg = str(e) if "429" in error_msg: - self.logger.warning(f"[DEBUG] Rate limited while cloning submission {submission.id}: {e}") - self.logger.warning(f"[DEBUG] Will continue with next submission instead of failing entire download") + self.logger.warning( + f"[DEBUG] Rate limited while cloning submission {submission.id}: {e}" + ) + self.logger.warning( + f"[DEBUG] Will continue with next submission instead of failing entire download" + ) continue else: # Not a rate limiting error, re-raise raise - + # Update processed count download_info["items_processed"] = processed_items - + except Exception as e: import traceback + error_msg = str(e) submission_id = submission.id if submission is not None else "unknown" stack_trace = traceback.format_exc() - + if "429" in error_msg: - self.logger.error(f"[DEBUG] Rate limited while cloning submission {submission_id}: {e}") + self.logger.error( + f"[DEBUG] Rate limited while cloning submission {submission_id}: {e}" + ) self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") self.logger.info(f"[DEBUG] Error message contains '429': {error_msg}") download_info["status"] = DownloadStatus.FAILED.value download_info["error"] = f"Rate limited by Reddit API: {e}" download_info["end_time"] = datetime.now() - + error_event = ProgressEvent( "error", download_id, f"Rate limited by Reddit API: {e}", None, - {"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"} + {"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"}, ) asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event)) return @@ -934,16 +986,18 @@ class BDFRManager: self.logger.error(f"[DEBUG] Error cloning submission {submission_id}: {e}") self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") continue - + self.logger.info(f"[DEBUG] Generator cloned {submission_count} submissions") - + # Final progress update if actual_count < limit: download_info["items_found"] = actual_count self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}") - - self.logger.info(f"[DEBUG] Final progress update - cloned {processed_items} out of {download_info['items_found']} found") - + + self.logger.info( + f"[DEBUG] Final progress update - cloned {processed_items} out of {download_info['items_found']} found" + ) + progress_event = ProgressEvent( "progress", download_id, @@ -952,11 +1006,11 @@ class BDFRManager: { "phase": "completed", "items_processed": processed_items, - "items_found": download_info["items_found"] - } + "items_found": download_info["items_found"], + }, ) asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) - + else: # For other downloaders, just run normally self.logger.info(f"[DEBUG] Using fallback downloader (no specific handler)") @@ -964,6 +1018,7 @@ class BDFRManager: except Exception as e: import traceback + stack_trace = traceback.format_exc() self.logger.error(f"Error in progress download for {download_id}: {e}") self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") @@ -971,7 +1026,7 @@ class BDFRManager: raise # Set up authentication if token provided - if hasattr(config, 'auth_token') and config.auth_token: + if hasattr(config, "auth_token") and config.auth_token: self.logger.info(f"[DEBUG] Setting up authenticated Reddit instance") # Monkey patch the create_reddit_instance method to use our token original_create_reddit_instance = downloader.create_reddit_instance @@ -985,11 +1040,12 @@ class BDFRManager: # Override the reddit_instance with authenticated version import praw + downloader.reddit_instance = praw.Reddit( client_id=downloader.cfg_parser.get("DEFAULT", "client_id"), client_secret=downloader.cfg_parser.get("DEFAULT", "client_secret"), - user_agent=getattr(downloader, 'user_agent', 'BDFR-Web-Interface/1.0'), - token=auth_token + user_agent=getattr(downloader, "user_agent", "BDFR-Web-Interface/1.0"), + token=auth_token, ) downloader.authenticated = True self.logger.info(f"[DEBUG] Created authenticated Reddit instance successfully") @@ -1005,10 +1061,14 @@ class BDFRManager: async def _notify_callbacks(self, callbacks: List[ProgressCallback], method_name: str, event: ProgressEvent): """Notify all callbacks of an event""" - self.logger.info(f"[BDFR-API] Notifying {len(callbacks)} callbacks of {method_name} event for download {event.download_id}") + self.logger.info( + f"[BDFR-API] Notifying {len(callbacks)} callbacks of {method_name} event for download {event.download_id}" + ) for callback in callbacks: try: - logger.info(f"[BDFR-API] Invoking {method_name} on callback {callback.__class__.__name__} for download {event.download_id}") + logger.info( + f"[BDFR-API] Invoking {method_name} on callback {callback.__class__.__name__} for download {event.download_id}" + ) method = getattr(callback, method_name) if asyncio.iscoroutinefunction(method): await method(event) @@ -1080,10 +1140,7 @@ class BDFRManager: Returns: Dictionary of download information keyed by download ID """ - return { - download_id: self.get_download_status(download_id) - for download_id in self._active_downloads - } + return {download_id: self.get_download_status(download_id) for download_id in self._active_downloads} def cleanup_completed(self, max_age_seconds: int = 3600) -> int: """ @@ -1099,7 +1156,11 @@ class BDFRManager: to_remove = [] for download_id, download_info in self._active_downloads.items(): - if download_info["status"] in [DownloadStatus.COMPLETED.value, DownloadStatus.FAILED.value, DownloadStatus.CANCELLED.value]: + if download_info["status"] in [ + DownloadStatus.COMPLETED.value, + DownloadStatus.FAILED.value, + DownloadStatus.CANCELLED.value, + ]: end_time = download_info.get("end_time") if end_time and (current_time - end_time).total_seconds() > max_age_seconds: to_remove.append(download_id) @@ -1124,7 +1185,7 @@ class BDFRManager: time_filter: str = "all", no_dupes: bool = False, simple_check: bool = False, - progress_callbacks: Optional[List[ProgressCallback]] = None + progress_callbacks: Optional[List[ProgressCallback]] = None, ) -> str: """ Download content from a subreddit. @@ -1168,12 +1229,7 @@ class BDFRManager: config.no_dupes = no_dupes config.simple_check = simple_check - download_id = self.create_download( - DownloadType.SUBREDDIT, - subreddit_name, - config, - progress_callbacks - ) + download_id = self.create_download(DownloadType.SUBREDDIT, subreddit_name, config, progress_callbacks) self.start_download(download_id) return download_id @@ -1187,7 +1243,7 @@ class BDFRManager: saved: bool = False, no_dupes: bool = False, simple_check: bool = False, - progress_callbacks: Optional[List[ProgressCallback]] = None + progress_callbacks: Optional[List[ProgressCallback]] = None, ) -> str: """ Download content from a user. @@ -1234,7 +1290,7 @@ class BDFRManager: config.saved = saved config.no_dupes = no_dupes config.simple_check = simple_check - + # Set authentication if token is available for this manager instance if self.auth_token: config.authenticate = True @@ -1244,12 +1300,7 @@ class BDFRManager: config.authenticate = False logger.info(f"[DEBUG] No authentication token available for user download") - download_id = self.create_download( - DownloadType.USER, - username, - config, - progress_callbacks - ) + download_id = self.create_download(DownloadType.USER, username, config, progress_callbacks) self.start_download(download_id) return download_id @@ -1260,7 +1311,7 @@ class BDFRManager: format_type: str = "json", limit: Optional[int] = None, simple_check: bool = False, - progress_callbacks: Optional[List[ProgressCallback]] = None + progress_callbacks: Optional[List[ProgressCallback]] = None, ) -> str: """ Archive subreddit data (metadata only, no downloads). @@ -1298,12 +1349,7 @@ class BDFRManager: config.limit = limit config.format = format_type - download_id = self.create_download( - DownloadType.ARCHIVE, - subreddit_name, - config, - progress_callbacks - ) + download_id = self.create_download(DownloadType.ARCHIVE, subreddit_name, config, progress_callbacks) self.start_download(download_id) return download_id @@ -1315,7 +1361,7 @@ class BDFRManager: format_type: str = "json", no_dupes: bool = False, simple_check: bool = False, - progress_callbacks: Optional[List[ProgressCallback]] = None + progress_callbacks: Optional[List[ProgressCallback]] = None, ) -> str: """ Clone subreddit (both download and archive). @@ -1355,12 +1401,7 @@ class BDFRManager: config.format = format_type config.no_dupes = no_dupes - download_id = self.create_download( - DownloadType.CLONE, - subreddit_name, - config, - progress_callbacks - ) + download_id = self.create_download(DownloadType.CLONE, subreddit_name, config, progress_callbacks) self.start_download(download_id) return download_id @@ -1394,4 +1435,4 @@ def set_bdfr_manager(manager: BDFRManager): manager: BDFRManager instance to use as default """ global _default_manager - _default_manager = manager \ No newline at end of file + _default_manager = manager diff --git a/bdfr/archiver.py b/bdfr/archiver.py index dec1c64..868dea5 100644 --- a/bdfr/archiver.py +++ b/bdfr/archiver.py @@ -52,7 +52,9 @@ class Archiver(RedditConnector): logger.error(f"Submission {submission.id} failed to be archived due to a PRAW exception: {e}") except prawcore.PrawcoreException as e: if submission is not None: - logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}") + logger.error( + f"The submission after {submission.id} failed to download due to a PRAW exception: {e}" + ) else: logger.error(f"Download failed due to a PRAW exception: {e}") logger.debug("Waiting 60 seconds to continue") diff --git a/bdfr/cloner.py b/bdfr/cloner.py index 8376199..e0271c9 100644 --- a/bdfr/cloner.py +++ b/bdfr/cloner.py @@ -30,7 +30,9 @@ class RedditCloner(RedditDownloader, Archiver): logger.error(f"Submission {submission.id} failed to be cloned due to a PRAW exception: {e}") except prawcore.PrawcoreException as e: if submission is not None: - logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}") + logger.error( + f"The submission after {submission.id} failed to download due to a PRAW exception: {e}" + ) else: logger.error(f"Download failed due to a PRAW exception: {e}") logger.debug("Waiting 60 seconds to continue") diff --git a/bdfr/connector.py b/bdfr/connector.py index a32dd5c..f3941cd 100644 --- a/bdfr/connector.py +++ b/bdfr/connector.py @@ -414,9 +414,13 @@ class RedditConnector(metaclass=ABCMeta): if TooManyRequests is not None and isinstance(e, TooManyRequests): is_rate_limited = True logger.info(f"Rate limited detected: Exception is TooManyRequests for user {user}") - elif (hasattr(e, "response") and getattr(e.response, "status_code", None) == 429) or "429" in str(e): + elif (hasattr(e, "response") and getattr(e.response, "status_code", None) == 429) or "429" in str( + e + ): is_rate_limited = True - logger.info(f"Rate limited detected: Status code 429 or '429' in error message for user {user}. Error: {e}") + logger.info( + f"Rate limited detected: Status code 429 or '429' in error message for user {user}. Error: {e}" + ) if is_rate_limited: logger.error("Received HTTP 429 (rate limited). Propagating error to fail the download.") @@ -441,7 +445,11 @@ class RedditConnector(metaclass=ABCMeta): def create_file_name_formatter(self) -> FileNameFormatter: return FileNameFormatter( - self.args.file_scheme, self.args.folder_scheme, self.args.time_format, self.args.filename_restriction_scheme, self.args.strip_unicode + self.args.file_scheme, + self.args.folder_scheme, + self.args.time_format, + self.args.filename_restriction_scheme, + self.args.strip_unicode, ) def create_time_filter(self) -> RedditTypes.TimeType: diff --git a/bdfr/downloader.py b/bdfr/downloader.py index 7783721..92b0b2f 100644 --- a/bdfr/downloader.py +++ b/bdfr/downloader.py @@ -50,11 +50,13 @@ class RedditDownloader(RedditConnector): hash_data = self._load_hash_list() # Handle both old and new hash file formats - if isinstance(hash_data, dict) and 'files' in hash_data: + if isinstance(hash_data, dict) and "files" in hash_data: # New format with enhanced structure - self.master_hash_list = {k: v['path'] for k, v in hash_data['files'].items()} - self.url_list = hash_data.get('urls', {}) - logger.info(f"Loaded {len(self.master_hash_list)} hashes and {len(self.url_list)} URLs from enhanced hash file") + self.master_hash_list = {k: v["path"] for k, v in hash_data["files"].items()} + self.url_list = hash_data.get("urls", {}) + logger.info( + f"Loaded {len(self.master_hash_list)} hashes and {len(self.url_list)} URLs from enhanced hash file" + ) else: # Old format - just hashes self.master_hash_list = hash_data @@ -70,8 +72,10 @@ class RedditDownloader(RedditConnector): if hash_value not in existing_hashes: self.master_hash_list[hash_value] = file_path - logger.info(f"Loaded {len(self.master_hash_list)} total hashes " - f"({len(existing_hashes)} from file, {len(all_files_hashes) - len(existing_hashes)} new)") + logger.info( + f"Loaded {len(self.master_hash_list)} total hashes " + f"({len(existing_hashes)} from file, {len(all_files_hashes) - len(existing_hashes)} new)" + ) def download(self): for generator in self.reddit_lists: @@ -164,7 +168,7 @@ class RedditDownloader(RedditConnector): self.master_hash_list[existing_file_hash] = destination # Store URL mapping for simple-check functionality if URL is available - if hasattr(res, 'url') and self.args.simple_check: + if hasattr(res, "url") and self.args.simple_check: self.url_list[res.url] = existing_file_hash logger.debug(f"Added hash for existing file: {existing_file_hash}") @@ -190,7 +194,7 @@ class RedditDownloader(RedditConnector): resource_hash = res.hash.hexdigest() # Simple-check: URL-based duplicate detection (fast path) - if self.args.simple_check and hasattr(res, 'url') and res.url in self.url_list: + if self.args.simple_check and hasattr(res, "url") and res.url in self.url_list: stored_hash = self.url_list[res.url] if stored_hash in self.master_hash_list: logger.info(f"URL {res.url} from submission {submission.id} already downloaded (simple-check)") @@ -233,7 +237,7 @@ class RedditDownloader(RedditConnector): self.master_hash_list[resource_hash] = destination # Store URL mapping for simple-check functionality - if hasattr(res, 'url') and self.args.simple_check: + if hasattr(res, "url") and self.args.simple_check: self.url_list[res.url] = resource_hash logger.debug(f"Hash added to master list: {resource_hash}") @@ -252,7 +256,7 @@ class RedditDownloader(RedditConnector): @staticmethod def scan_existing_files(directory: Path) -> dict[str, Path]: files = [] - for (dirpath, _dirnames, filenames) in os.walk(directory): + for dirpath, _dirnames, filenames in os.walk(directory): files.extend([Path(dirpath, file) for file in filenames]) logger.info(f"Calculating hashes for {len(files)} files") @@ -270,14 +274,14 @@ class RedditDownloader(RedditConnector): def _load_hash_list(self) -> dict[str, Path]: """Load existing hash list from .bdfr_hashes.json in download directory.""" logger.debug(f"Loading hash list from directory: {self.download_directory}") - hash_file_path = self.download_directory / '.bdfr_hashes.json' + hash_file_path = self.download_directory / ".bdfr_hashes.json" if not hash_file_path.exists(): logger.debug(f"No existing hash file found at {hash_file_path}") return {} try: - with open(hash_file_path, 'r', encoding='utf-8') as f: + with open(hash_file_path, "r", encoding="utf-8") as f: hash_data = json.load(f) if not isinstance(hash_data, dict): @@ -285,16 +289,16 @@ class RedditDownloader(RedditConnector): return {} # Handle new enhanced format - if 'files' in hash_data and isinstance(hash_data['files'], dict): + if "files" in hash_data and isinstance(hash_data["files"], dict): # New format with enhanced structure - files_data = hash_data['files'] + files_data = hash_data["files"] loaded_hashes = {} - urls_data = hash_data.get('urls', {}) + urls_data = hash_data.get("urls", {}) for hash_value, file_info in files_data.items(): - if isinstance(file_info, dict) and 'path' in file_info: + if isinstance(file_info, dict) and "path" in file_info: # New format: {"hash": {"path": "relative/path", "url": "http://..."}} - relative_path = file_info['path'] + relative_path = file_info["path"] absolute_path = self.download_directory / relative_path if absolute_path.exists(): loaded_hashes[hash_value] = absolute_path @@ -302,8 +306,8 @@ class RedditDownloader(RedditConnector): logger.debug(f"File {absolute_path} from hash file no longer exists") # Load URL mapping for simple-check - if 'url' in file_info and file_info['url']: - self.url_list[file_info['url']] = hash_value + if "url" in file_info and file_info["url"]: + self.url_list[file_info["url"]] = hash_value elif isinstance(file_info, str): # Legacy format within new structure: {"hash": "relative/path"} absolute_path = self.download_directory / file_info @@ -337,30 +341,30 @@ class RedditDownloader(RedditConnector): def _save_hash_list(self) -> None: """Save current hash list to .bdfr_hashes.json in download directory using atomic write.""" - hash_file_path = self.download_directory / '.bdfr_hashes.json' + hash_file_path = self.download_directory / ".bdfr_hashes.json" # Build enhanced data structure for new format if self.args.simple_check: # New enhanced format with URLs and metadata hash_data = { - 'files': {}, - 'urls': self.url_list.copy(), - 'metadata': { - 'version': '2.0', - 'created_with': 'simple_check' if self.args.simple_check else 'standard', - 'url_count': len(self.url_list), - 'hash_count': len(self.master_hash_list) - } + "files": {}, + "urls": self.url_list.copy(), + "metadata": { + "version": "2.0", + "created_with": "simple_check" if self.args.simple_check else "standard", + "url_count": len(self.url_list), + "hash_count": len(self.master_hash_list), + }, } # Convert absolute paths to relative paths for portability for hash_value, absolute_path in self.master_hash_list.items(): try: relative_path = absolute_path.relative_to(self.download_directory) - hash_data['files'][hash_value] = { - 'path': str(relative_path), - 'url': next((url for url, h in self.url_list.items() if h == hash_value), None), - 'check_method': 'hash' + hash_data["files"][hash_value] = { + "path": str(relative_path), + "url": next((url for url, h in self.url_list.items() if h == hash_value), None), + "check_method": "hash", } except ValueError: # File is not relative to download directory, skip it @@ -381,17 +385,13 @@ class RedditDownloader(RedditConnector): # Atomic write: write to temporary file first, then rename try: with tempfile.NamedTemporaryFile( - mode='w', - dir=self.download_directory, - suffix='.tmp', - delete=False, - encoding='utf-8' + mode="w", dir=self.download_directory, suffix=".tmp", delete=False, encoding="utf-8" ) as temp_file: json.dump(hash_data, temp_file, indent=2) temp_file_path = temp_file.name # Atomic rename - if os.name == 'nt': # Windows + if os.name == "nt": # Windows # On Windows, we need to remove the target file first if it exists if hash_file_path.exists(): hash_file_path.unlink() @@ -407,7 +407,7 @@ class RedditDownloader(RedditConnector): logger.error(f"Unexpected error saving hash file {hash_file_path}: {e}") # Clean up temp file if it still exists try: - if 'temp_file_path' in locals(): + if "temp_file_path" in locals(): os.unlink(temp_file_path) except (OSError, IOError): pass diff --git a/bdfr/examples/api_usage.py b/bdfr/examples/api_usage.py index 42acd40..f697368 100644 --- a/bdfr/examples/api_usage.py +++ b/bdfr/examples/api_usage.py @@ -19,20 +19,16 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from bdfr.api import ( BDFRManager, - DownloadType, DownloadStatus, - ProgressEvent, - ProgressCallback, + DownloadType, LoggingCallback, - get_bdfr_manager + ProgressCallback, + ProgressEvent, + get_bdfr_manager, ) # Configure logging -logging.basicConfig( - level=logging.INFO, - format='[%(asctime)s] %(levelname)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S' -) +logging.basicConfig(level=logging.INFO, format="[%(asctime)s] %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") logger = logging.getLogger(__name__) @@ -53,7 +49,7 @@ class WebSocketCallback(ProgressCallback): async def on_error(self, event: ProgressEvent): """Send error update to WebSocket""" print(f"āŒ [{self.websocket_id}] ERROR: {event.message}") - if event.data.get('exception'): + if event.data.get("exception"): print(f" Exception: {event.data['exception']}") async def on_completed(self, event: ProgressEvent): @@ -95,9 +91,9 @@ async def example_basic_usage(): # Create a download for a subreddit download_id = manager.download_subreddit( "python", # subreddit name - limit=10, # download 10 posts - sort="hot", # sort by hot - no_dupes=True # avoid duplicates + limit=10, # download 10 posts + sort="hot", # sort by hot + no_dupes=True, # avoid duplicates ) print(f"šŸ“‹ Created download with ID: {download_id}") @@ -111,7 +107,7 @@ async def example_basic_usage(): print(f"šŸ“Š Status: {status['status']} | Progress: {int(round(status['progress']))}%") - if status['status'] in ['completed', 'failed', 'cancelled']: + if status["status"] in ["completed", "failed", "cancelled"]: print(f"šŸ Download finished with status: {status['status']}") break @@ -126,11 +122,7 @@ async def example_advanced_usage(): print("=" * 50) # Create custom callbacks - callbacks = [ - LoggingCallback("web_interface"), - WebSocketCallback("user_123"), - DatabaseCallback() - ] + callbacks = [LoggingCallback("web_interface"), WebSocketCallback("user_123"), DatabaseCallback()] # Create manager with custom download directory manager = BDFRManager("./custom_downloads") @@ -140,11 +132,7 @@ async def example_advanced_usage(): download_ids = [] for subreddit in subreddits: - download_id = manager.create_download( - DownloadType.SUBREDDIT, - subreddit, - progress_callbacks=callbacks - ) + download_id = manager.create_download(DownloadType.SUBREDDIT, subreddit, progress_callbacks=callbacks) # Start the download manager.start_download(download_id) @@ -164,7 +152,7 @@ async def example_advanced_usage(): print(f"šŸ“Š {download_id}: {status['status']} ({int(round(status['progress']))}%)") - if status['status'] in ['completed', 'failed', 'cancelled']: + if status["status"] in ["completed", "failed", "cancelled"]: print(f"šŸ Download {download_id} finished") download_ids.remove(download_id) else: @@ -187,11 +175,7 @@ async def example_user_download(): # Download user's submitted posts download_id = manager.download_user( - "testuser", # username - limit=25, # 25 posts - submitted=True, - upvoted=False, - saved=False + "testuser", limit=25, submitted=True, upvoted=False, saved=False # username # 25 posts ) print(f"šŸ“‹ Created user download: {download_id}") @@ -205,7 +189,7 @@ async def example_user_download(): print(f"šŸ“Š Status: {status['status']} | Progress: {int(round(status['progress']))}%") - if status['status'] in ['completed', 'failed']: + if status["status"] in ["completed", "failed"]: break await asyncio.sleep(2) @@ -221,11 +205,7 @@ async def example_archive_operation(): manager = BDFRManager("./archives") # Archive subreddit data (metadata only) - download_id = manager.archive_subreddit( - "dataisbeautiful", - format_type="json", - limit=50 - ) + download_id = manager.archive_subreddit("dataisbeautiful", format_type="json", limit=50) print(f"šŸ“‹ Created archive operation: {download_id}") @@ -238,7 +218,7 @@ async def example_archive_operation(): print(f"šŸ“Š Archive status: {status['status']} | Progress: {int(round(status['progress']))}%") - if status['status'] in ['completed', 'failed']: + if status["status"] in ["completed", "failed"]: print(f"šŸ Archive finished with status: {status['status']}") break @@ -265,11 +245,7 @@ async def example_web_integration(): callback = WebSocketCallback(f"ws_{user_id}") # Create and start download - download_id = self.manager.download_subreddit( - subreddit, - limit=limit, - progress_callbacks=[callback] - ) + download_id = self.manager.download_subreddit(subreddit, limit=limit, progress_callbacks=[callback]) # Track for this user session if user_id not in self.active_sessions: @@ -279,7 +255,7 @@ async def example_web_integration(): return { "success": True, "download_id": download_id, - "message": f"Started download of r/{subreddit} (limit: {limit})" + "message": f"Started download of r/{subreddit} (limit: {limit})", } async def get_user_downloads(self, user_id: str): @@ -329,7 +305,7 @@ async def example_web_integration(): # Cancel one download if user_downloads: - cancel_result = await app.cancel_user_download(user_id, user_downloads[0]['id']) + cancel_result = await app.cancel_user_download(user_id, user_downloads[0]["id"]) print(f"Cancel result: {cancel_result}") return len(user_downloads) @@ -343,10 +319,7 @@ async def example_error_handling(): manager = BDFRManager("./test_downloads") # Try to download from a non-existent subreddit - download_id = manager.download_subreddit( - "this_subreddit_does_not_exist", - limit=5 - ) + download_id = manager.download_subreddit("this_subreddit_does_not_exist", limit=5) print(f"šŸ“‹ Created download for non-existent subreddit: {download_id}") @@ -359,7 +332,7 @@ async def example_error_handling(): print(f"šŸ“Š Status: {status['status']}") - if status['status'] == 'failed': + if status["status"] == "failed": print(f"šŸ Download failed as expected: {status.get('error', 'Unknown error')}") break @@ -401,9 +374,10 @@ async def main(): except Exception as e: print(f"\nāŒ Error running examples: {e}") import traceback + traceback.print_exc() if __name__ == "__main__": # Run the examples - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) diff --git a/bdfr/file_name_formatter.py b/bdfr/file_name_formatter.py index 3df6829..2fd36e6 100644 --- a/bdfr/file_name_formatter.py +++ b/bdfr/file_name_formatter.py @@ -245,10 +245,26 @@ class FileNameFormatter: if ord(char) < 0x80: result.append(char) # Keep common Unicode letters, numbers, and punctuation - elif unicodedata.category(char) in ['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Ps', 'Pe', 'Pi', 'Pf', 'Po']: + elif unicodedata.category(char) in [ + "Lu", + "Ll", + "Lt", + "Lm", + "Lo", + "Nd", + "Nl", + "No", + "Pc", + "Pd", + "Ps", + "Pe", + "Pi", + "Pf", + "Po", + ]: result.append(char) # Strip emoji, symbols, and other special characters that cause 8.3 names - elif unicodedata.category(char).startswith(('S', 'So', 'Sk', 'Sm')): # Symbols + elif unicodedata.category(char).startswith(("S", "So", "Sk", "Sm")): # Symbols continue elif ord(char) > 0x1F000: # High Unicode ranges often contain emoji continue @@ -256,7 +272,7 @@ class FileNameFormatter: # Keep other Unicode characters that are generally safe result.append(char) - return ''.join(result) + return "".join(result) @staticmethod def _strip_emojis(input_string: str) -> str: diff --git a/bdfr/resource.py b/bdfr/resource.py index 2eec13f..38d743a 100644 --- a/bdfr/resource.py +++ b/bdfr/resource.py @@ -24,11 +24,11 @@ class Resource: self.content: Optional[bytes] = None self.url = url self.hash: Optional[_hashlib.HASH] = None - + # Log the original extension before normalization if extension: logger.debug(f"Resource constructor received extension: '{extension}' for URL: {url}") - + self.extension = self._normalize_extension(extension) self.download_function = download_function if not self.extension: @@ -72,14 +72,14 @@ class Resource: if self.url.startswith("https://www.reddit.com/media"): logger.debug(f"Detected Reddit media URL: {self.url}") parsed_url = urllib.parse.urlparse(self.url) - url_param = urllib.parse.parse_qs(parsed_url.query).get('url', [None])[0] + url_param = urllib.parse.parse_qs(parsed_url.query).get("url", [None])[0] if url_param: decoded_url = urllib.parse.unquote(url_param) logger.debug(f"Reddit media URL decoded to: {decoded_url}") stripped_url = urllib.parse.urlsplit(decoded_url).path # Also handle preview.redd.it URLs which might not have extensions - elif "preview.redd.it" in self.url and not stripped_url.endswith(('.jpg', '.jpeg', '.png', '.gif', '.webp')): + elif "preview.redd.it" in self.url and not stripped_url.endswith((".jpg", ".jpeg", ".png", ".gif", ".webp")): logger.debug(f"Detected preview.redd.it URL without extension: {self.url}") # For preview URLs, try to infer from common patterns or add fallback logic @@ -92,7 +92,7 @@ class Resource: logger.warning(f"Could not determine extension for URL: {self.url} (path: {stripped_url})") # As a last resort, if we have content, try to detect by magic numbers - if hasattr(self, 'content') and self.content: + if hasattr(self, "content") and self.content: detected = self._detect_extension_by_content() return self._normalize_extension(detected) if detected else None @@ -104,21 +104,21 @@ class Resource: return None # Check for common image formats - if self.content.startswith(b'\xFF\xD8\xFF'): + if self.content.startswith(b"\xff\xd8\xff"): logger.debug(f"Detected JPEG by magic number for URL: {self.url}") - return '.jpg' - elif self.content.startswith(b'\x89PNG\r\n\x1a\n'): + return ".jpg" + elif self.content.startswith(b"\x89PNG\r\n\x1a\n"): logger.debug(f"Detected PNG by magic number for URL: {self.url}") - return '.png' - elif self.content.startswith(b'GIF87a') or self.content.startswith(b'GIF89a'): + return ".png" + elif self.content.startswith(b"GIF87a") or self.content.startswith(b"GIF89a"): logger.debug(f"Detected GIF by magic number for URL: {self.url}") - return '.gif' - elif self.content.startswith(b'RIFF') and self.content[8:12] == b'WEBP': + return ".gif" + elif self.content.startswith(b"RIFF") and self.content[8:12] == b"WEBP": logger.debug(f"Detected WebP by magic number for URL: {self.url}") - return '.webp' - elif self.content.startswith(b'BM'): + return ".webp" + elif self.content.startswith(b"BM"): logger.debug(f"Detected BMP by magic number for URL: {self.url}") - return '.bmp' + return ".bmp" logger.debug(f"Could not detect file type by magic number for URL: {self.url}") return None @@ -127,15 +127,17 @@ class Resource: """Normalize extension to lowercase for consistency""" if not extension: return None - + original = extension # Ensure extension starts with a dot - if not extension.startswith('.'): - extension = '.' + extension - + if not extension.startswith("."): + extension = "." + extension + normalized = extension.lower() if original != normalized: - logger.info(f"Extension normalization: '{original}' -> '{normalized}' for URL: {self.url if hasattr(self, 'url') else 'unknown'}") + logger.info( + f"Extension normalization: '{original}' -> '{normalized}' for URL: {self.url if hasattr(self, 'url') else 'unknown'}" + ) return normalized @staticmethod diff --git a/bdfr/site_downloaders/download_factory.py b/bdfr/site_downloaders/download_factory.py index 6687200..31c0dd5 100644 --- a/bdfr/site_downloaders/download_factory.py +++ b/bdfr/site_downloaders/download_factory.py @@ -25,6 +25,7 @@ class DownloadFactory: @staticmethod def pull_lever(url: str) -> type[BaseDownloader]: import logging + logger = logging.getLogger(__name__) sanitised_url = DownloadFactory.sanitise_url(url).lower() diff --git a/tests/test_duplicate_fix.py b/tests/test_duplicate_fix.py index b34a999..5eb6ccd 100644 --- a/tests/test_duplicate_fix.py +++ b/tests/test_duplicate_fix.py @@ -4,14 +4,15 @@ Test script to verify that the duplicate folder creation fix works correctly. This script simulates the scenario where duplicate posts would previously create empty folders. """ -import tempfile import shutil -from pathlib import Path -from unittest.mock import MagicMock # Add the bdfr module to the path import sys -sys.path.insert(0, '.') +import tempfile +from pathlib import Path +from unittest.mock import MagicMock + +sys.path.insert(0, ".") from bdfr.configuration import Configuration from bdfr.connector import RedditConnector @@ -69,6 +70,7 @@ def test_duplicate_folder_creation_fix(): # Mock the download factory import bdfr.site_downloaders.download_factory as df + original_pull_lever = df.DownloadFactory.pull_lever df.DownloadFactory.pull_lever = MagicMock(return_value=mock_downloader_class) @@ -93,4 +95,4 @@ def test_duplicate_folder_creation_fix(): if __name__ == "__main__": test_duplicate_folder_creation_fix() - print("All tests passed! The duplicate folder creation fix is working correctly.") \ No newline at end of file + print("All tests passed! The duplicate folder creation fix is working correctly.") diff --git a/tests/test_extension_case_normalization.py b/tests/test_extension_case_normalization.py index 01b25ac..bdf25a0 100644 --- a/tests/test_extension_case_normalization.py +++ b/tests/test_extension_case_normalization.py @@ -3,8 +3,10 @@ Test extension case normalization functionality """ -import pytest from unittest.mock import MagicMock + +import pytest + from bdfr.resource import Resource @@ -43,7 +45,9 @@ class TestExtensionNormalization: mock_submission.id = "test123" resource = Resource(mock_submission, url, lambda: None) - assert resource.extension == expected, f"Reddit media URL {url} should normalize to {expected}, got {resource.extension}" + assert ( + resource.extension == expected + ), f"Reddit media URL {url} should normalize to {expected}, got {resource.extension}" def test_constructor_extensions_normalized(self): """Test that extensions passed to constructor are normalized""" @@ -66,7 +70,9 @@ class TestExtensionNormalization: mock_submission.id = "test123" resource = Resource(mock_submission, "https://example.com/test", lambda: None, input_ext) - assert resource.extension == expected, f"Constructor extension {input_ext} should normalize to {expected}, got {resource.extension}" + assert ( + resource.extension == expected + ), f"Constructor extension {input_ext} should normalize to {expected}, got {resource.extension}" def test_magic_number_detection_normalized(self): """Test that magic number detection returns normalized extensions""" @@ -74,7 +80,7 @@ class TestExtensionNormalization: mock_submission.id = "test123" # Test JPEG magic number detection - jpeg_content = b'\xFF\xD8\xFF' + b'0' * 100 # JPEG magic number + jpeg_content = b"\xff\xd8\xff" + b"0" * 100 # JPEG magic number resource = Resource(mock_submission, "https://example.com/no-extension", lambda params: jpeg_content) resource.download() # Trigger content-based detection - assert resource.extension == ".jpg", f"Magic number detection should return .jpg, got {resource.extension}" \ No newline at end of file + assert resource.extension == ".jpg", f"Magic number detection should return .jpg, got {resource.extension}" diff --git a/tests/test_extension_debug.py b/tests/test_extension_debug.py index a05e0c6..2926c40 100644 --- a/tests/test_extension_debug.py +++ b/tests/test_extension_debug.py @@ -3,13 +3,16 @@ Test script to debug file extension detection issues """ -import sys import os +import sys + sys.path.insert(0, os.path.dirname(__file__)) from unittest.mock import MagicMock + from bdfr.resource import Resource + def test_extension_detection(): """Test extension detection with various URL patterns""" @@ -18,26 +21,20 @@ def test_extension_detection(): ("https://example.com/image.jpg", ".jpg"), ("https://example.com/video.mp4", ".mp4"), ("https://files.example.com/document.pdf", ".pdf"), - # URLs without extensions ("https://example.com/api/data", None), ("https://example.com/path/without/extension", None), - # URLs with query parameters ("https://example.com/image.jpg?size=large", ".jpg"), ("https://example.com/video.mp4?utm_source=test", ".mp4"), - # URLs with fragments ("https://example.com/image.png#section", ".png"), - # Complex paths ("https://imgur.com/a/gallery123", None), ("https://reddit.com/r/test/abc123_def456_789", None), - # Edge cases that might cause weird names ("https://example.com/L7SW9E~G", None), ("https://example.com/temp/file", None), - # Reddit media URLs (the actual issue) ("https://www.reddit.com/media?url=https%3A%2F%2Fi.redd.it%2Fr2mv10i4vkfd1.jpeg", ".jpeg"), ("https://i.redd.it/r2mv10i4vkfd1.jpeg", ".jpeg"), @@ -60,5 +57,6 @@ def test_extension_detection(): print(f"Match: {'YES' if resource.extension == expected else 'NO'}") print("-" * 40) + if __name__ == "__main__": - test_extension_detection() \ No newline at end of file + test_extension_detection() diff --git a/tests/test_file_locking_fix.py b/tests/test_file_locking_fix.py index e86f58e..1061c83 100644 --- a/tests/test_file_locking_fix.py +++ b/tests/test_file_locking_fix.py @@ -7,16 +7,18 @@ This script simulates the scenario where a download fails and then tries to redo import asyncio import logging import os + +# Add the bdfr module to the path +import sys import tempfile import time from pathlib import Path -# Add the bdfr module to the path -import sys sys.path.insert(0, str(Path(__file__).parent)) from bdfr.api import BDFRManager, DownloadType, LoggingCallback, ProgressEvent + class TestProgressCallback(LoggingCallback): """Test callback that simulates a failure""" @@ -36,6 +38,7 @@ class TestProgressCallback(LoggingCallback): self.events.append(event) await super().on_completed(event) + def test_file_locking_fix(): """Test that the file locking issue is resolved""" print("Testing file locking fix...") @@ -53,7 +56,7 @@ def test_file_locking_fix(): download_id1 = manager.create_download( DownloadType.USER, "test_user_12345", # This user doesn't exist, should fail - progress_callbacks=[TestProgressCallback()] + progress_callbacks=[TestProgressCallback()], ) # Start the download (it should fail) @@ -71,7 +74,7 @@ def test_file_locking_fix(): download_id2 = manager.create_download( DownloadType.USER, "test_user_67890", # This user also doesn't exist, should fail - progress_callbacks=[TestProgressCallback()] + progress_callbacks=[TestProgressCallback()], ) # Start the second download @@ -100,7 +103,7 @@ def test_file_locking_fix(): print(f" - {log_file.name}") # Check if file is accessible (not locked) try: - with open(log_file, 'r') as f: + with open(log_file, "r") as f: content = f.read() print(f" SUCCESS: Log file is accessible ({len(content)} characters)") except PermissionError: @@ -112,9 +115,7 @@ def test_file_locking_fix(): # Test 4: Try to create a third download to ensure cleanup worked print("\n4. Creating third download to verify cleanup...") download_id3 = manager.create_download( - DownloadType.USER, - "test_user_cleanup", - progress_callbacks=[TestProgressCallback()] + DownloadType.USER, "test_user_cleanup", progress_callbacks=[TestProgressCallback()] ) success3 = manager.start_download(download_id3) @@ -132,6 +133,7 @@ def test_file_locking_fix(): print("\nSUCCESS: All tests passed! File locking issue appears to be fixed.") return True + if __name__ == "__main__": try: success = test_file_locking_fix() @@ -144,5 +146,6 @@ if __name__ == "__main__": except Exception as e: print(f"\nTest failed with exception: {e}") import traceback + traceback.print_exc() - sys.exit(1) \ No newline at end of file + sys.exit(1) diff --git a/tests/test_file_name_formatter.py b/tests/test_file_name_formatter.py index 23fee9b..ade31e2 100644 --- a/tests/test_file_name_formatter.py +++ b/tests/test_file_name_formatter.py @@ -538,7 +538,7 @@ def test_strip_unicode_chars(input_string: str, expected: str): def test_unicode_stripping_enabled(submission: MagicMock): """Test that Unicode stripping is applied when enabled""" - submission.title = 'Test šŸ’• emoji' + submission.title = "Test šŸ’• emoji" formatter = FileNameFormatter("{TITLE}", "", "", strip_unicode=True) result = formatter._format_name(submission, "{TITLE}") assert "šŸ’•" not in result @@ -547,7 +547,7 @@ def test_unicode_stripping_enabled(submission: MagicMock): def test_unicode_stripping_disabled(submission: MagicMock): """Test that Unicode stripping is not applied when disabled""" - submission.title = 'Test šŸ’• emoji' + submission.title = "Test šŸ’• emoji" formatter = FileNameFormatter("{TITLE}", "", "", strip_unicode=False) result = formatter._format_name(submission, "{TITLE}") assert "šŸ’•" in result diff --git a/tests/test_hash_persistence.py b/tests/test_hash_persistence.py index 5fd147d..48ca789 100644 --- a/tests/test_hash_persistence.py +++ b/tests/test_hash_persistence.py @@ -3,14 +3,15 @@ Test script to verify hash persistence functionality. """ import json -import tempfile import shutil -from pathlib import Path -from unittest.mock import Mock # Import the necessary modules import sys -sys.path.insert(0, '/Users/Daniel/Documents/GitHub/bulk-downloader-for-reddit') +import tempfile +from pathlib import Path +from unittest.mock import Mock + +sys.path.insert(0, "/Users/Daniel/Documents/GitHub/bulk-downloader-for-reddit") from bdfr.configuration import Configuration from bdfr.downloader import RedditDownloader @@ -59,7 +60,7 @@ def test_hash_persistence(): # Test 2: Save empty hash list print("Test 2: Saving empty hash list") downloader._save_hash_list() - hash_file = temp_path / '.bdfr_hashes.json' + hash_file = temp_path / ".bdfr_hashes.json" assert hash_file.exists(), "Hash file should be created even when empty" print("PASS Passed") @@ -71,18 +72,18 @@ def test_hash_persistence(): # Test 4: Add some test data and save print("Test 4: Adding test data and saving") - test_file = temp_path / 'test.txt' + test_file = temp_path / "test.txt" test_file.write_text("test content") - downloader.master_hash_list['test_hash_123'] = test_file + downloader.master_hash_list["test_hash_123"] = test_file downloader._save_hash_list() # Verify the saved JSON structure - with open(hash_file, 'r') as f: + with open(hash_file, "r") as f: saved_data = json.load(f) - assert 'test_hash_123' in saved_data, "Test hash should be in saved data" - assert saved_data['test_hash_123'] == 'test.txt', f"Expected 'test.txt', got {saved_data['test_hash_123']}" + assert "test_hash_123" in saved_data, "Test hash should be in saved data" + assert saved_data["test_hash_123"] == "test.txt", f"Expected 'test.txt', got {saved_data['test_hash_123']}" print("PASS Passed") # Test 5: Load hash list and verify data is restored @@ -100,13 +101,15 @@ def test_hash_persistence(): loaded_hash_list = new_downloader._load_hash_list() assert len(loaded_hash_list) == 1, f"Expected 1 hash, got {len(loaded_hash_list)}" - assert 'test_hash_123' in loaded_hash_list, "Test hash should be loaded" - assert loaded_hash_list['test_hash_123'] == test_file, f"File path should match: {loaded_hash_list['test_hash_123']} != {test_file}" + assert "test_hash_123" in loaded_hash_list, "Test hash should be loaded" + assert ( + loaded_hash_list["test_hash_123"] == test_file + ), f"File path should match: {loaded_hash_list['test_hash_123']} != {test_file}" print("PASS Passed") # Test 6: Test corrupted hash file handling print("Test 6: Testing corrupted hash file handling") - with open(hash_file, 'w') as f: + with open(hash_file, "w") as f: f.write("invalid json content") corrupted_downloader = RedditDownloader.__new__(RedditDownloader) @@ -122,7 +125,9 @@ def test_hash_persistence(): # Should handle corrupted file gracefully and return empty dict corrupted_hash_list = corrupted_downloader._load_hash_list() - assert len(corrupted_hash_list) == 0, f"Expected empty hash list for corrupted file, got {len(corrupted_hash_list)}" + assert ( + len(corrupted_hash_list) == 0 + ), f"Expected empty hash list for corrupted file, got {len(corrupted_hash_list)}" print("PASS Passed") print("\nAll tests passed! Hash persistence functionality is working correctly.") @@ -174,7 +179,7 @@ def test_simple_check_functionality(): # Test 2: Add test data and save with simple_check format print("Test 2: Adding test data and saving with simple_check format") - test_file = temp_path / 'test.txt' + test_file = temp_path / "test.txt" test_file.write_text("test content") test_url = "https://example.com/test.txt" test_hash = "test_hash_123" @@ -185,16 +190,16 @@ def test_simple_check_functionality(): downloader._save_hash_list() # Verify the saved JSON structure has enhanced format - with open(temp_path / '.bdfr_hashes.json', 'r') as f: + with open(temp_path / ".bdfr_hashes.json", "r") as f: saved_data = json.load(f) - assert 'files' in saved_data, "Enhanced format should have 'files' section" - assert 'urls' in saved_data, "Enhanced format should have 'urls' section" - assert 'metadata' in saved_data, "Enhanced format should have 'metadata' section" - assert test_hash in saved_data['files'], "Test hash should be in files section" - assert test_url in saved_data['urls'], "Test URL should be in urls section" - assert saved_data['metadata']['version'] == '2.0', "Version should be 2.0" - assert saved_data['metadata']['created_with'] == 'simple_check', "Should be created with simple_check" + assert "files" in saved_data, "Enhanced format should have 'files' section" + assert "urls" in saved_data, "Enhanced format should have 'urls' section" + assert "metadata" in saved_data, "Enhanced format should have 'metadata' section" + assert test_hash in saved_data["files"], "Test hash should be in files section" + assert test_url in saved_data["urls"], "Test URL should be in urls section" + assert saved_data["metadata"]["version"] == "2.0", "Version should be 2.0" + assert saved_data["metadata"]["created_with"] == "simple_check", "Should be created with simple_check" print("PASS") # Test 3: Load hash list and verify URL mapping is restored @@ -228,7 +233,7 @@ def test_simple_check_functionality(): mock_resource.hash.hexdigest.return_value = test_hash # Create a mock destination that exists - mock_destination = temp_path / 'existing_file.txt' + mock_destination = temp_path / "existing_file.txt" mock_destination.parent.mkdir(parents=True, exist_ok=True) mock_destination.write_text("existing content") @@ -260,17 +265,14 @@ def test_backward_compatibility(): temp_path = Path(temp_dir) # Create old-format hash file manually - (temp_path / 'relative' / 'path').mkdir(parents=True, exist_ok=True) - (temp_path / 'relative' / 'path' / 'file1.txt').write_text("content1") - (temp_path / 'relative' / 'path' / 'file2.txt').write_text("content2") + (temp_path / "relative" / "path").mkdir(parents=True, exist_ok=True) + (temp_path / "relative" / "path" / "file1.txt").write_text("content1") + (temp_path / "relative" / "path" / "file2.txt").write_text("content2") - old_hash_data = { - "hash1": "relative/path/file1.txt", - "hash2": "relative/path/file2.txt" - } + old_hash_data = {"hash1": "relative/path/file1.txt", "hash2": "relative/path/file2.txt"} - hash_file = temp_path / '.bdfr_hashes.json' - with open(hash_file, 'w') as f: + hash_file = temp_path / ".bdfr_hashes.json" + with open(hash_file, "w") as f: json.dump(old_hash_data, f) # Create downloader and load old format @@ -294,21 +296,21 @@ def test_backward_compatibility(): print("PASS - Old format loaded correctly") # Test saving in new format - (temp_path / 'another').mkdir(parents=True, exist_ok=True) - test_file = temp_path / 'another' / 'new_file.txt' + (temp_path / "another").mkdir(parents=True, exist_ok=True) + test_file = temp_path / "another" / "new_file.txt" test_file.write_text("new content") downloader.master_hash_list["new_hash"] = test_file downloader._save_hash_list() # Verify new format was created - with open(hash_file, 'r') as f: + with open(hash_file, "r") as f: new_data = json.load(f) - assert 'files' in new_data, "New format should have 'files' section" - assert 'urls' in new_data, "New format should have 'urls' section" - assert 'metadata' in new_data, "New format should have 'metadata' section" - assert new_data['metadata']['version'] == '2.0', "Should be version 2.0" + assert "files" in new_data, "New format should have 'files' section" + assert "urls" in new_data, "New format should have 'urls' section" + assert "metadata" in new_data, "New format should have 'metadata' section" + assert new_data["metadata"]["version"] == "2.0", "Should be version 2.0" print("PASS - Old format upgraded to new format correctly") @@ -318,4 +320,4 @@ def test_backward_compatibility(): if __name__ == "__main__": test_hash_persistence() test_simple_check_functionality() - test_backward_compatibility() \ No newline at end of file + test_backward_compatibility() diff --git a/tests/test_user_folder_structure.py b/tests/test_user_folder_structure.py index f0b0ed8..cef1244 100644 --- a/tests/test_user_folder_structure.py +++ b/tests/test_user_folder_structure.py @@ -14,10 +14,11 @@ import tempfile from pathlib import Path # Set UTF-8 encoding for Windows console -if sys.platform == 'win32': +if sys.platform == "win32": import codecs - sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer, 'strict') - sys.stderr = codecs.getwriter('utf-8')(sys.stderr.buffer, 'strict') + + sys.stdout = codecs.getwriter("utf-8")(sys.stdout.buffer, "strict") + sys.stderr = codecs.getwriter("utf-8")(sys.stderr.buffer, "strict") from bdfr.api import BDFRManager, DownloadType from bdfr.configuration import Configuration @@ -26,27 +27,24 @@ from bdfr.configuration import Configuration def test_subreddit_directory_structure(): """Test that subreddit downloads use correct directory structure""" print("\n=== Testing Subreddit Directory Structure ===") - + with tempfile.TemporaryDirectory() as tmpdir: manager = BDFRManager(download_directory=tmpdir) - + # Create a download for a subreddit - download_id = manager.create_download( - DownloadType.SUBREDDIT, - "test_subreddit" - ) - + download_id = manager.create_download(DownloadType.SUBREDDIT, "test_subreddit") + download_info = manager.get_download_status(download_id) config = download_info["config"] - + expected_dir = str(Path(tmpdir)) actual_dir = config.directory - + print(f"Expected directory: {expected_dir}") print(f"Actual directory: {actual_dir}") print(f"Subreddit config: {config.subreddit}") print(f"Folder scheme: {config.folder_scheme}") - + assert actual_dir == expected_dir, f"Subreddit directory mismatch!" print("[OK] Subreddit directory structure is correct") print(f" Files will be saved to: {actual_dir}/{{SUBREDDIT}}/{{files}}") @@ -55,28 +53,25 @@ def test_subreddit_directory_structure(): def test_user_directory_structure(): """Test that user downloads use correct directory structure""" print("\n=== Testing User Directory Structure ===") - + with tempfile.TemporaryDirectory() as tmpdir: manager = BDFRManager(download_directory=tmpdir) - + # Create a download for a user username = "test_user" - download_id = manager.create_download( - DownloadType.USER, - username - ) - + download_id = manager.create_download(DownloadType.USER, username) + download_info = manager.get_download_status(download_id) config = download_info["config"] - + expected_dir = str(Path(tmpdir) / username) actual_dir = config.directory - + print(f"Expected directory: {expected_dir}") print(f"Actual directory: {actual_dir}") print(f"User config: {config.user}") print(f"Folder scheme: {config.folder_scheme}") - + assert actual_dir == expected_dir, f"User directory mismatch!" print("[OK] User directory structure is correct") print(f" Files will be saved to: {actual_dir}/{{SUBREDDIT}}/{{files}}") @@ -85,26 +80,26 @@ def test_user_directory_structure(): def test_convenience_method(): """Test the convenience method download_user()""" print("\n=== Testing download_user() Convenience Method ===") - + with tempfile.TemporaryDirectory() as tmpdir: manager = BDFRManager(download_directory=tmpdir) - + # Don't actually start the download, just check the config username = "convenience_test_user" - + # Create config manually like the convenience method does config = Configuration() user_directory = Path(tmpdir) / username config.directory = str(user_directory) config.user = [username] - + expected_dir = str(Path(tmpdir) / username) actual_dir = config.directory - + print(f"Expected directory: {expected_dir}") print(f"Actual directory: {actual_dir}") print(f"User config: {config.user}") - + assert actual_dir == expected_dir, f"Convenience method directory mismatch!" print("[OK] Convenience method directory structure is correct") @@ -118,7 +113,7 @@ def demonstrate_folder_structure(): print(" ā”œā”€ā”€ file1.jpg") print(" ā”œā”€ā”€ file2.png") print(" └── file3.mp4") - + print("\nWhen downloading from a USER 'spez' who posts to multiple subreddits:") print(" downloads/") print(" └── spez/") @@ -129,7 +124,7 @@ def demonstrate_folder_structure(): print(" │ └── file3.jpg") print(" └── pics/") print(" └── file4.png") - + print("\n[OK] This structure allows:") print(" 1. Easy identification of user-specific downloads") print(" 2. Organization by subreddit within each user folder") @@ -140,22 +135,23 @@ if __name__ == "__main__": print("=" * 60) print("Testing User Folder Structure Changes") print("=" * 60) - + try: test_subreddit_directory_structure() test_user_directory_structure() test_convenience_method() demonstrate_folder_structure() - + print("\n" + "=" * 60) print("[SUCCESS] All tests passed!") print("=" * 60) - + except AssertionError as e: print(f"\n[FAIL] Test failed: {e}") exit(1) except Exception as e: print(f"\n[ERROR] Unexpected error: {e}") import traceback + traceback.print_exc() - exit(1) \ No newline at end of file + exit(1)