#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ BDFR API Layer This module provides a direct integration interface for the web interface, eliminating the need for subprocess console parsing. It wraps the existing BDFR core classes and provides a clean API with structured progress callbacks. """ import asyncio import logging import os import tempfile import threading import time import uuid from abc import ABC, abstractmethod from collections.abc import Callable, Iterable from datetime import datetime from enum import Enum from pathlib import Path from typing import Any, Dict, List, Optional, Union from bdfr.configuration import Configuration from bdfr.connector import RedditConnector from bdfr.downloader import RedditDownloader from bdfr.archiver import Archiver from bdfr.cloner import RedditCloner from bdfr import exceptions as errors logger = logging.getLogger(__name__) class DownloadType(Enum): """Types of downloads supported by BDFR""" SUBREDDIT = "subreddit" USER = "user" MULTIREDDIT = "multireddit" SUBMISSIONS = "submissions" ARCHIVE = "archive" CLONE = "clone" class DownloadStatus(Enum): """Status of a download operation""" QUEUED = "queued" RUNNING = "running" COMPLETED = "completed" FAILED = "failed" CANCELLED = "cancelled" PAUSED = "paused" class ProgressEvent: """Represents a progress event with structured data""" def __init__( self, event_type: str, download_id: str, message: str, progress: Optional[float] = None, data: Optional[Dict[str, Any]] = None, timestamp: Optional[datetime] = None ): self.event_type = event_type # "progress", "status", "error", "completed" self.download_id = download_id self.message = message self.progress = progress # 0-100 percentage self.data = data or {} self.timestamp = timestamp or datetime.now() def to_dict(self) -> Dict[str, Any]: """Convert to dictionary for JSON serialization""" return { "event_type": self.event_type, "download_id": self.download_id, "message": self.message, "progress": self.progress, "data": self.data, "timestamp": self.timestamp.isoformat() } class ProgressCallback(ABC): """Abstract base class for progress callbacks""" @abstractmethod async def on_progress(self, event: ProgressEvent): """Called when progress is made""" pass @abstractmethod async def on_error(self, event: ProgressEvent): """Called when an error occurs""" pass @abstractmethod async def on_completed(self, event: ProgressEvent): """Called when download is completed""" pass class LoggingCallback(ProgressCallback): """Default logging-based progress callback""" def __init__(self, logger_name: str = __name__): self.logger = logging.getLogger(logger_name) async def on_progress(self, event: ProgressEvent): """Log progress events""" if event.progress is not None: self.logger.info(f"[{event.download_id}] {event.message} ({int(round(event.progress))}%)") else: self.logger.info(f"[{event.download_id}] {event.message}") async def on_error(self, event: ProgressEvent): """Log error events""" self.logger.error(f"[{event.download_id}] ERROR: {event.message}") if event.data.get('exception'): self.logger.exception(f"[{event.download_id}] Exception details", exc_info=event.data['exception']) async def on_completed(self, event: ProgressEvent): """Log completion events""" if event.event_type == "completed": self.logger.info(f"[{event.download_id}] {event.message}") else: self.logger.info(f"[{event.download_id}] {event.message}") class BDFRManager: """ Main API interface for BDFR operations. This class provides a clean interface for the web application to interact with BDFR functionality without needing subprocess calls. """ def __init__(self, download_directory: Optional[Union[str, Path]] = None, auth_token: Optional[str] = None): """ Initialize the BDFR Manager. Args: download_directory: Base directory for downloads. If None, uses current directory. auth_token: Optional authentication token for Reddit API """ self.download_directory = Path(download_directory or ".").resolve() self.download_directory.mkdir(exist_ok=True, parents=True) self.auth_token = auth_token # Active downloads tracking self._active_downloads: Dict[str, Dict[str, Any]] = {} self._download_threads: Dict[str, threading.Thread] = {} self._callbacks: Dict[str, List[ProgressCallback]] = {} # Setup logging first self.logger = logging.getLogger(__name__) # Default configuration self._default_config = self._create_default_config() # Override the log file path to avoid conflicts between multiple BDFR instances if not self._default_config.log: # Create a unique log file for this BDFR manager instance log_dir = self.download_directory / "logs" log_dir.mkdir(exist_ok=True) unique_log_file = log_dir / f"bdfr_manager_{int(time.time())}_{id(self)}.log" self._default_config.log = str(unique_log_file) def _create_default_config(self) -> Configuration: """Create a default configuration for BDFR operations""" config = Configuration() config.directory = str(self.download_directory) config.limit = 100 # Default limit config.sort = "hot" config.time = "all" config.no_dupes = False config.make_hard_links = False config.verbose = 1 # Set authentication if token provided if self.auth_token: config.authenticate = True self.logger.info(f"[DEBUG] Authentication enabled for download with token") else: config.authenticate = False self.logger.info(f"[DEBUG] No authentication token provided, using anonymous access") # Archiver defaults config.format = "json" config.all_comments = False config.comment_context = False return config def create_download( self, download_type: DownloadType, name: str, config: Optional[Configuration] = None, progress_callbacks: Optional[List[ProgressCallback]] = None ) -> str: """ Create a new download operation. Args: download_type: Type of download (subreddit, user, etc.) name: Name of the target (subreddit name, username, etc.) config: BDFR configuration. If None, uses defaults. progress_callbacks: List of progress callbacks. If None, uses logging callback. Returns: Download ID for tracking the operation """ download_id = str(uuid.uuid4()) # Use provided config or default if config is None: # Create a new Configuration instance with default values config = Configuration() config.directory = str(self.download_directory) config.limit = 100 # Default limit config.sort = "hot" config.time = "all" config.no_dupes = False config.make_hard_links = False config.verbose = 1 config.format = "json" config.all_comments = False config.comment_context = False # Create unique log file for this download log_dir = self.download_directory / "logs" log_dir.mkdir(exist_ok=True) unique_log_file = log_dir / f"bdfr_{download_type.value}_{name}_{download_id}_{int(time.time())}.log" config.log = str(unique_log_file) # Set specific parameters based on download type if download_type == DownloadType.SUBREDDIT: config.subreddit = [name] elif download_type == DownloadType.USER: config.user = [name] # Append username to directory path for user downloads # This creates folder structure: downloads/username/subreddit/files user_directory = self.download_directory / name config.directory = str(user_directory) elif download_type == DownloadType.MULTIREDDIT: config.multireddit = [name] # Setup callbacks if progress_callbacks is None: progress_callbacks = [LoggingCallback()] self._callbacks[download_id] = progress_callbacks # Initialize download tracking self._active_downloads[download_id] = { "id": download_id, "type": download_type.value, "name": name, "status": DownloadStatus.QUEUED.value, "progress": 0.0, "start_time": datetime.now(), "config": config, "callbacks": progress_callbacks, "items_processed": 0, "items_found": 0, "current_item": None, "phase": "queued" } self.logger.info(f"Created download {download_id} for {download_type.value}: {name}") return download_id def start_download(self, download_id: str) -> bool: """ Start a download operation asynchronously. Args: download_id: ID of the download to start Returns: True if started successfully, False if download not found or already running """ if download_id not in self._active_downloads: self.logger.error(f"Download {download_id} not found") return False download_info = self._active_downloads[download_id] if download_info["status"] == DownloadStatus.RUNNING.value: self.logger.warning(f"Download {download_id} is already running") return False # Update status to running download_info["status"] = DownloadStatus.RUNNING.value download_info["start_time"] = datetime.now() # Create and start download thread download_thread = threading.Thread( target=self._run_download, args=(download_id,), daemon=True ) self._download_threads[download_id] = download_thread download_thread.start() self.logger.info(f"Started download {download_id}") return True def _run_download(self, download_id: str): """Internal method to run a download in a separate thread""" download_info = self._active_downloads[download_id] config = download_info["config"] callbacks = download_info["callbacks"] downloader = None try: self.logger.info(f"[DEBUG] Starting download {download_id} for {download_info['type']}: {download_info['name']}") self.logger.info(f"[DEBUG] Config: subreddit={getattr(config, 'subreddit', None)}, user={getattr(config, 'user', None)}, limit={getattr(config, 'limit', None)}") # Create progress event for start start_event = ProgressEvent( "status", download_id, f"Starting {download_info['type']} download: {download_info['name']}", 0.0, {"phase": "starting"} ) # Notify callbacks self.logger.info(f"[BDFR-API] Notifying {len(callbacks)} callbacks of progress event for download {download_id}") asyncio.run(self._notify_callbacks(callbacks, "on_progress", start_event)) # Set up authentication token in config if provided if self.auth_token: # Store the token in the config for the connector to use config.auth_token = self.auth_token self.logger.info(f"[DEBUG] Set auth token in config") # Ensure unique log file for this download to avoid conflicts if not config.log: # Create a unique log file for this download attempt log_dir = self.download_directory / "logs" log_dir.mkdir(exist_ok=True) unique_log_file = log_dir / f"bdfr_download_{download_id}_{int(time.time())}.log" config.log = str(unique_log_file) # Ensure log directory exists before creating downloader if config.log: log_path = Path(config.log) log_path.parent.mkdir(parents=True, exist_ok=True) # Determine which BDFR class to use based on configuration self.logger.info(f"[DEBUG] Determining downloader type - submitted={getattr(config, 'submitted', False)}, upvoted={getattr(config, 'upvoted', False)}, saved={getattr(config, 'saved', False)}, format={getattr(config, 'format', 'N/A')}") try: if config.submitted or config.upvoted or config.saved: # User data download self.logger.info(f"[DEBUG] Creating RedditDownloader for user data") downloader = RedditDownloader(config, []) download_info["operation_type"] = "user_download" self.logger.info(f"[DEBUG] Using RedditDownloader for user data") elif hasattr(config, 'format') and config.format in ["json", "xml", "yaml"]: # Archive operation self.logger.info(f"[DEBUG] Creating Archiver for archive operation") downloader = Archiver(config, []) download_info["operation_type"] = "archive" self.logger.info(f"[DEBUG] Using Archiver for archive operation") else: # Standard download self.logger.info(f"[DEBUG] Creating RedditDownloader for standard download") downloader = RedditDownloader(config, []) download_info["operation_type"] = "download" self.logger.info(f"[DEBUG] Using RedditDownloader for standard download") except Exception as e: self.logger.error(f"[DEBUG] Failed to create downloader: {e}") raise # Run the actual download with progress tracking self.logger.info(f"[DEBUG] Starting download_with_progress for {download_id}") try: self._download_with_progress(download_id, downloader, callbacks, config) except Exception as e: self.logger.error(f"[DEBUG] Exception in download_with_progress: {e}") raise # Mark as completed only if not already failed or cancelled if download_info.get("status") not in [DownloadStatus.FAILED.value, DownloadStatus.CANCELLED.value]: self.logger.info(f"[DEBUG] Marking download {download_id} as completed") self.logger.info(f"[DEBUG] Final stats - items_processed: {download_info['items_processed']}, items_found: {download_info['items_found']}") download_info["status"] = DownloadStatus.COMPLETED.value download_info["progress"] = 100.0 download_info["end_time"] = datetime.now() completed_event = ProgressEvent( "completed", download_id, f"Download completed successfully", 100.0, { "items_processed": download_info["items_processed"], "items_found": download_info["items_found"], "operation_type": download_info.get("operation_type") } ) self.logger.info(f"[DEBUG] Sending completion callback for {download_id}") self.logger.info(f"[BDFR-API] Sending completion event for download {download_id}") asyncio.run(self._notify_callbacks(callbacks, "on_completed", completed_event)) else: self.logger.info(f"[DEBUG] Skipping completion for {download_id} due to status={download_info.get('status')}") except Exception as e: import traceback # Get full stack trace for debugging stack_trace = traceback.format_exc() # Determine if this failure was due to rate limiting (HTTP 429) is_rate_limited = False try: import prawcore TooManyRequests = getattr(prawcore.exceptions, "TooManyRequests", None) if TooManyRequests is not None and isinstance(e, TooManyRequests): is_rate_limited = True except Exception: pass if "429" in str(e): is_rate_limited = True # Mark as failed download_info["status"] = DownloadStatus.FAILED.value download_info["error"] = str(e) download_info["stack_trace"] = stack_trace download_info["end_time"] = datetime.now() error_event = ProgressEvent( "error", download_id, f"Download failed: {str(e)}", None, {"exception": str(e), "stack_trace": stack_trace, "phase": ("rate_limited" if is_rate_limited else "failed")} ) asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event)) finally: # Clean up thread reference if download_id in self._download_threads: del self._download_threads[download_id] # Clean up logging handlers to prevent file locking issues if downloader is not None: try: # Close any logging handlers that might be holding file locks logger = logging.getLogger() for handler in logger.handlers[:]: if hasattr(handler, 'close') and hasattr(handler, 'baseFilename'): # Check if this handler is for our download's log file if hasattr(config, 'log') and config.log: handler_path = getattr(handler, 'baseFilename', '') if handler_path == config.log or handler_path.startswith(str(self.download_directory)): try: handler.close() logger.removeHandler(handler) self.logger.info(f"[DEBUG] Closed logging handler for {handler_path}") except Exception as cleanup_error: self.logger.warning(f"[DEBUG] Error closing handler {handler_path}: {cleanup_error}") except Exception as cleanup_error: self.logger.warning(f"[DEBUG] Error during logging cleanup: {cleanup_error}") def _download_with_progress(self, download_id: str, downloader, callbacks: List[ProgressCallback], config): """Run download with progress tracking and callbacks""" download_info = self._active_downloads[download_id] # Create a progress-tracking wrapper for the downloader original_download = downloader.download def progress_download(): """Wrapper for download method that sends progress updates""" try: # Initialize submission variable before any potential exceptions submission = None # Update phase to fetching download_info["phase"] = "fetching_submissions" progress_event = ProgressEvent( "progress", download_id, "Fetching submissions...", 10.0, {"phase": "fetching_submissions"} ) self.logger.info(f"[BDFR-API] Sending progress update for download {download_id}") asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) # For RedditDownloader, we need to intercept the download process if hasattr(downloader, 'reddit_lists') and hasattr(downloader, '_download_submission'): processed_submissions = 0 self.logger.info(f"[DEBUG] RedditDownloader has reddit_lists and _download_submission methods") # Get the limit from config - this is our target/goal for items_found # items_found represents "how many items are available to download (up to limit)" # items_processed represents "how many we've actually processed so far" limit = getattr(config, 'limit', 100) download_info["items_found"] = limit self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}") # Process submissions with progress tracking self.logger.info(f"[DEBUG] Starting submission processing loop...") actual_count = 0 # Track actual submissions found for final adjustment for generator in downloader.reddit_lists: self.logger.info(f"[DEBUG] Processing generator...") submission_count = 0 for submission in generator: # Ensure submission is defined for the entire loop if submission is None: continue submission_count += 1 actual_count += 1 if download_info["status"] == DownloadStatus.CANCELLED.value: self.logger.info(f"Download {download_id} was cancelled") return try: self.logger.info(f"[DEBUG] Processing submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}") # Update current item being processed download_info["current_item"] = submission.id download_info["current_item_type"] = "submission" download_info["current_subreddit"] = submission.subreddit.display_name # Send progress update for this submission processed_submissions += 1 # Calculate progress based on processed vs expected (limit), but cap at 90% # This gives smooth progress even when we don't know the exact total yet progress_percent = 20.0 + (processed_submissions / max(limit, 1)) * 70.0 # 20-90% range progress_event = ProgressEvent( "progress", download_id, f"Processing submission: {submission.id}", min(progress_percent, 90.0), { "phase": "downloading_submission", "current_item": submission.id, "current_item_type": "submission", "current_subreddit": submission.subreddit.display_name, "items_processed": processed_submissions, "items_found": download_info["items_found"] # Keep showing the limit as target } ) asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) # Process this submission self.logger.info(f"[DEBUG] Calling _download_submission for {submission.id}") downloader._download_submission(submission) self.logger.info(f"[DEBUG] Completed _download_submission for {submission.id}") # Update processed count download_info["items_processed"] = processed_submissions except Exception as e: import traceback error_msg = str(e) submission_id = submission.id if submission is not None else "unknown" # Get full stack trace for debugging stack_trace = traceback.format_exc() if "429" in error_msg: self.logger.error(f"[DEBUG] Rate limited while processing submission {submission_id}: {e}") self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") # Mark download as failed due to rate limiting download_info["status"] = DownloadStatus.FAILED.value download_info["error"] = f"Rate limited by Reddit API: {e}" download_info["end_time"] = datetime.now() error_event = ProgressEvent( "error", download_id, f"Rate limited by Reddit API: {e}", None, {"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"} ) asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event)) return else: self.logger.error(f"[DEBUG] Error processing submission {submission_id}: {e}") self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") continue self.logger.info(f"[DEBUG] Generator processed {submission_count} submissions") # Final progress update # If we found fewer items than the limit, update items_found to reflect reality if actual_count < limit: download_info["items_found"] = actual_count self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}") self.logger.info(f"[DEBUG] Final progress update - processed {processed_submissions} out of {download_info['items_found']} found") progress_event = ProgressEvent( "progress", download_id, "Download completed", 100.0, { "phase": "completed", "items_processed": processed_submissions, "items_found": download_info["items_found"] } ) asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) elif hasattr(downloader, 'reddit_lists') and hasattr(downloader, 'write_entry'): # For Archiver, we need to intercept the archiving process processed_items = 0 self.logger.info(f"[DEBUG] Archiver detected with reddit_lists and write_entry methods") # Get the limit from config limit = getattr(config, 'limit', 100) download_info["items_found"] = limit self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}") # Process items with progress tracking self.logger.info(f"[DEBUG] Starting archiving loop...") actual_count = 0 for generator in downloader.reddit_lists: self.logger.info(f"[DEBUG] Processing generator for archiving...") item_count = 0 for item in generator: if item is None: continue item_count += 1 actual_count += 1 if download_info["status"] == DownloadStatus.CANCELLED.value: self.logger.info(f"Download {download_id} was cancelled") return try: # Determine item type import praw.models if isinstance(item, praw.models.Submission): item_id = item.id item_type = "submission" subreddit_name = item.subreddit.display_name elif isinstance(item, praw.models.Comment): item_id = item.id item_type = "comment" subreddit_name = item.subreddit.display_name if hasattr(item, 'subreddit') else "unknown" else: item_id = str(item) item_type = "item" subreddit_name = "unknown" self.logger.info(f"[DEBUG] Archiving {item_type} {item_count}: {item_id} from r/{subreddit_name}") # Update current item being processed download_info["current_item"] = item_id download_info["current_item_type"] = item_type download_info["current_subreddit"] = subreddit_name # Send progress update for this item processed_items += 1 progress_percent = 20.0 + (processed_items / max(limit, 1)) * 70.0 progress_event = ProgressEvent( "progress", download_id, f"Archiving {item_type}: {item_id}", min(progress_percent, 90.0), { "phase": "archiving_item", "current_item": item_id, "current_item_type": item_type, "current_subreddit": subreddit_name, "items_processed": processed_items, "items_found": download_info["items_found"] } ) asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) # Archive this item self.logger.info(f"[DEBUG] Calling write_entry for {item_id}") downloader.write_entry(item) self.logger.info(f"[DEBUG] Completed write_entry for {item_id}") # Update processed count download_info["items_processed"] = processed_items except Exception as e: import traceback error_msg = str(e) item_id = item_id if 'item_id' in locals() else "unknown" stack_trace = traceback.format_exc() if "429" in error_msg: self.logger.error(f"[DEBUG] Rate limited while archiving item {item_id}: {e}") self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") download_info["status"] = DownloadStatus.FAILED.value download_info["error"] = f"Rate limited by Reddit API: {e}" download_info["end_time"] = datetime.now() error_event = ProgressEvent( "error", download_id, f"Rate limited by Reddit API: {e}", None, {"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"} ) asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event)) return else: self.logger.error(f"[DEBUG] Error archiving item {item_id}: {e}") self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") continue self.logger.info(f"[DEBUG] Generator archived {item_count} items") # Final progress update if actual_count < limit: download_info["items_found"] = actual_count self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}") self.logger.info(f"[DEBUG] Final progress update - archived {processed_items} out of {download_info['items_found']} found") progress_event = ProgressEvent( "progress", download_id, "Archiving completed", 100.0, { "phase": "completed", "items_processed": processed_items, "items_found": download_info["items_found"] } ) asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) elif hasattr(downloader, 'reddit_lists') and hasattr(downloader, '_download_submission') and hasattr(downloader, 'write_entry'): # For RedditCloner (which has both download and archive capabilities) processed_items = 0 self.logger.info(f"[DEBUG] RedditCloner detected with reddit_lists, _download_submission, and write_entry methods") # Get the limit from config limit = getattr(config, 'limit', 100) download_info["items_found"] = limit self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}") # Process submissions with progress tracking (both download and archive) self.logger.info(f"[DEBUG] Starting cloning loop...") actual_count = 0 for generator in downloader.reddit_lists: self.logger.info(f"[DEBUG] Processing generator for cloning...") submission_count = 0 for submission in generator: if submission is None: continue submission_count += 1 actual_count += 1 if download_info["status"] == DownloadStatus.CANCELLED.value: self.logger.info(f"Download {download_id} was cancelled") return try: self.logger.info(f"[DEBUG] Cloning submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}") # Update current item being processed download_info["current_item"] = submission.id download_info["current_item_type"] = "submission" download_info["current_subreddit"] = submission.subreddit.display_name # Send progress update for this submission processed_items += 1 progress_percent = 20.0 + (processed_items / max(limit, 1)) * 70.0 progress_event = ProgressEvent( "progress", download_id, f"Cloning submission: {submission.id}", min(progress_percent, 90.0), { "phase": "cloning_submission", "current_item": submission.id, "current_item_type": "submission", "current_subreddit": submission.subreddit.display_name, "items_processed": processed_items, "items_found": download_info["items_found"] } ) asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) # Clone this submission (download + archive) self.logger.info(f"[DEBUG] Calling _download_submission for {submission.id}") downloader._download_submission(submission) self.logger.info(f"[DEBUG] Calling write_entry for {submission.id}") downloader.write_entry(submission) self.logger.info(f"[DEBUG] Completed cloning for {submission.id}") # Update processed count download_info["items_processed"] = processed_items except Exception as e: import traceback error_msg = str(e) submission_id = submission.id if submission is not None else "unknown" stack_trace = traceback.format_exc() if "429" in error_msg: self.logger.error(f"[DEBUG] Rate limited while cloning submission {submission_id}: {e}") self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") download_info["status"] = DownloadStatus.FAILED.value download_info["error"] = f"Rate limited by Reddit API: {e}" download_info["end_time"] = datetime.now() error_event = ProgressEvent( "error", download_id, f"Rate limited by Reddit API: {e}", None, {"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"} ) asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event)) return else: self.logger.error(f"[DEBUG] Error cloning submission {submission_id}: {e}") self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") continue self.logger.info(f"[DEBUG] Generator cloned {submission_count} submissions") # Final progress update if actual_count < limit: download_info["items_found"] = actual_count self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}") self.logger.info(f"[DEBUG] Final progress update - cloned {processed_items} out of {download_info['items_found']} found") progress_event = ProgressEvent( "progress", download_id, "Cloning completed", 100.0, { "phase": "completed", "items_processed": processed_items, "items_found": download_info["items_found"] } ) asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) else: # For other downloaders, just run normally self.logger.info(f"[DEBUG] Using fallback downloader (no specific handler)") original_download() except Exception as e: import traceback stack_trace = traceback.format_exc() self.logger.error(f"Error in progress download for {download_id}: {e}") self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") raise # Set up authentication if token provided if hasattr(config, 'auth_token') and config.auth_token: self.logger.info(f"[DEBUG] Setting up authenticated Reddit instance") # Monkey patch the create_reddit_instance method to use our token original_create_reddit_instance = downloader.create_reddit_instance auth_token = config.auth_token # Capture token in closure def authenticated_create_reddit_instance(): try: self.logger.info(f"[DEBUG] Creating authenticated Reddit instance") # Call original method first to set up basic config original_create_reddit_instance() # Override the reddit_instance with authenticated version import praw downloader.reddit_instance = praw.Reddit( client_id=downloader.cfg_parser.get("DEFAULT", "client_id"), client_secret=downloader.cfg_parser.get("DEFAULT", "client_secret"), user_agent=getattr(downloader, 'user_agent', 'BDFR-Web-Interface/1.0'), token=auth_token ) downloader.authenticated = True self.logger.info(f"[DEBUG] Created authenticated Reddit instance successfully") except Exception as e: self.logger.error(f"[DEBUG] Failed to create authenticated Reddit instance: {e}") raise downloader.create_reddit_instance = authenticated_create_reddit_instance # Replace the download method and run it downloader.download = progress_download downloader.download() async def _notify_callbacks(self, callbacks: List[ProgressCallback], method_name: str, event: ProgressEvent): """Notify all callbacks of an event""" self.logger.info(f"[BDFR-API] Notifying {len(callbacks)} callbacks of {method_name} event for download {event.download_id}") for callback in callbacks: try: logger.info(f"[BDFR-API] Invoking {method_name} on callback {callback.__class__.__name__} for download {event.download_id}") method = getattr(callback, method_name) if asyncio.iscoroutinefunction(method): await method(event) else: # Run sync method in thread pool loop = asyncio.get_event_loop() await loop.run_in_executor(None, lambda: asyncio.run(method(event))) logger.info(f"[BDFR-API] Successfully invoked {method_name} on callback {callback.__class__.__name__}") except Exception as e: self.logger.error(f"[BDFR-API] Error in callback {callback.__class__.__name__}: {e}") def get_download_status(self, download_id: str) -> Optional[Dict[str, Any]]: """ Get the current status of a download. Args: download_id: ID of the download to check Returns: Download status information or None if not found """ if download_id not in self._active_downloads: return None download_info = dict(self._active_downloads[download_id]) # Add thread status if running if download_id in self._download_threads: thread = self._download_threads[download_id] download_info["thread_alive"] = thread.is_alive() return download_info def cancel_download(self, download_id: str) -> bool: """ Cancel a running download. Args: download_id: ID of the download to cancel Returns: True if cancelled successfully, False if download not found or not running """ if download_id not in self._active_downloads: return False download_info = self._active_downloads[download_id] if download_info["status"] not in [DownloadStatus.RUNNING.value, DownloadStatus.QUEUED.value]: return False # Mark as cancelled download_info["status"] = DownloadStatus.CANCELLED.value download_info["end_time"] = datetime.now() # Try to stop the thread (this is a best effort) if download_id in self._download_threads: thread = self._download_threads[download_id] # Note: We can't actually kill the thread safely, but we can mark it as cancelled # The BDFR instance would need to check for cancellation periodically self.logger.info(f"Cancelled download {download_id}") return True def list_downloads(self) -> Dict[str, Dict[str, Any]]: """ List all downloads (active and recently completed). Returns: Dictionary of download information keyed by download ID """ return { download_id: self.get_download_status(download_id) for download_id in self._active_downloads } def cleanup_completed(self, max_age_seconds: int = 3600) -> int: """ Clean up completed/failed downloads older than specified age. Args: max_age_seconds: Maximum age in seconds for completed downloads to keep Returns: Number of downloads cleaned up """ current_time = datetime.now() to_remove = [] for download_id, download_info in self._active_downloads.items(): if download_info["status"] in [DownloadStatus.COMPLETED.value, DownloadStatus.FAILED.value, DownloadStatus.CANCELLED.value]: end_time = download_info.get("end_time") if end_time and (current_time - end_time).total_seconds() > max_age_seconds: to_remove.append(download_id) for download_id in to_remove: del self._active_downloads[download_id] if download_id in self._callbacks: del self._callbacks[download_id] if to_remove: self.logger.info(f"Cleaned up {len(to_remove)} old downloads: {to_remove}") return len(to_remove) # Convenience methods for common operations def download_subreddit( self, subreddit_name: str, limit: Optional[int] = None, sort: str = "hot", time_filter: str = "all", no_dupes: bool = False, simple_check: bool = False, progress_callbacks: Optional[List[ProgressCallback]] = None ) -> str: """ Download content from a subreddit. Args: subreddit_name: Name of the subreddit (without r/) limit: Maximum number of posts to download sort: Sort method (hot, top, new, controversial, rising) time_filter: Time filter (all, hour, day, week, month, year) no_dupes: Whether to avoid duplicate downloads simple_check: Whether to use simple URL-based duplicate checking progress_callbacks: Optional progress callbacks Returns: Download ID for tracking """ # Create a new Configuration instance with default values config = Configuration() config.directory = str(self.download_directory) config.limit = 100 # Default limit config.sort = "hot" config.time = "all" config.no_dupes = False config.make_hard_links = False config.verbose = 1 config.format = "json" config.all_comments = False config.comment_context = False # Create unique log file for this download log_dir = self.download_directory / "logs" log_dir.mkdir(exist_ok=True) unique_log_file = log_dir / f"bdfr_subreddit_{subreddit_name}_{int(time.time())}.log" config.log = str(unique_log_file) config.subreddit = [subreddit_name] if limit: config.limit = limit config.sort = sort config.time = time_filter config.no_dupes = no_dupes config.simple_check = simple_check download_id = self.create_download( DownloadType.SUBREDDIT, subreddit_name, config, progress_callbacks ) self.start_download(download_id) return download_id def download_user( self, username: str, limit: Optional[int] = None, submitted: bool = True, upvoted: bool = False, saved: bool = False, no_dupes: bool = False, simple_check: bool = False, progress_callbacks: Optional[List[ProgressCallback]] = None ) -> str: """ Download content from a user. Args: username: Reddit username limit: Maximum number of posts to download submitted: Download user's submitted posts upvoted: Download user's upvoted posts (requires authentication) saved: Download user's saved posts (requires authentication) no_dupes: Whether to avoid duplicate downloads progress_callbacks: Optional progress callbacks Returns: Download ID for tracking """ # Create a new Configuration instance with default values config = Configuration() # Append username to directory path for user downloads # This creates folder structure: downloads/username/subreddit/files user_directory = self.download_directory / username config.directory = str(user_directory) config.limit = 100 # Default limit config.sort = "hot" config.time = "all" config.no_dupes = False config.make_hard_links = False config.verbose = 1 config.format = "json" config.all_comments = False config.comment_context = False # Create unique log file for this download log_dir = self.download_directory / "logs" log_dir.mkdir(exist_ok=True) unique_log_file = log_dir / f"bdfr_user_{username}_{int(time.time())}.log" config.log = str(unique_log_file) config.user = [username] if limit: config.limit = limit config.submitted = submitted config.upvoted = upvoted config.saved = saved config.no_dupes = no_dupes config.simple_check = simple_check download_id = self.create_download( DownloadType.USER, username, config, progress_callbacks ) self.start_download(download_id) return download_id def archive_subreddit( self, subreddit_name: str, format_type: str = "json", limit: Optional[int] = None, simple_check: bool = False, progress_callbacks: Optional[List[ProgressCallback]] = None ) -> str: """ Archive subreddit data (metadata only, no downloads). Args: subreddit_name: Name of the subreddit format_type: Archive format (json, xml, yaml) limit: Maximum number of posts to archive progress_callbacks: Optional progress callbacks Returns: Download ID for tracking """ # Create a new Configuration instance with default values config = Configuration() config.directory = str(self.download_directory) config.limit = 100 # Default limit config.sort = "hot" config.time = "all" config.no_dupes = False config.make_hard_links = False config.verbose = 1 config.format = "json" config.all_comments = False config.comment_context = False # Create unique log file for this download log_dir = self.download_directory / "logs" log_dir.mkdir(exist_ok=True) unique_log_file = log_dir / f"bdfr_archive_{subreddit_name}_{int(time.time())}.log" config.log = str(unique_log_file) config.subreddit = [subreddit_name] if limit: config.limit = limit config.format = format_type download_id = self.create_download( DownloadType.ARCHIVE, subreddit_name, config, progress_callbacks ) self.start_download(download_id) return download_id def clone_subreddit( self, subreddit_name: str, limit: Optional[int] = None, format_type: str = "json", no_dupes: bool = False, simple_check: bool = False, progress_callbacks: Optional[List[ProgressCallback]] = None ) -> str: """ Clone subreddit (both download and archive). Args: subreddit_name: Name of the subreddit limit: Maximum number of posts to process format_type: Archive format for metadata no_dupes: Whether to avoid duplicate downloads progress_callbacks: Optional progress callbacks Returns: Download ID for tracking """ # Create a new Configuration instance with default values config = Configuration() config.directory = str(self.download_directory) config.limit = 100 # Default limit config.sort = "hot" config.time = "all" config.no_dupes = False config.make_hard_links = False config.verbose = 1 config.format = "json" config.all_comments = False config.comment_context = False # Create unique log file for this download log_dir = self.download_directory / "logs" log_dir.mkdir(exist_ok=True) unique_log_file = log_dir / f"bdfr_clone_{subreddit_name}_{int(time.time())}.log" config.log = str(unique_log_file) config.subreddit = [subreddit_name] if limit: config.limit = limit config.format = format_type config.no_dupes = no_dupes download_id = self.create_download( DownloadType.CLONE, subreddit_name, config, progress_callbacks ) self.start_download(download_id) return download_id # Global instance for easy access _default_manager: Optional[BDFRManager] = None def get_bdfr_manager(download_directory: Optional[Union[str, Path]] = None) -> BDFRManager: """ Get the default BDFR manager instance. Args: download_directory: Base directory for downloads Returns: BDFRManager instance """ global _default_manager if _default_manager is None: _default_manager = BDFRManager(download_directory) return _default_manager def set_bdfr_manager(manager: BDFRManager): """ Set the default BDFR manager instance. Args: manager: BDFRManager instance to use as default """ global _default_manager _default_manager = manager