1388 lines
62 KiB
Python
1388 lines
62 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
"""
|
|
BDFR API Layer
|
|
|
|
This module provides a direct integration interface for the web interface,
|
|
eliminating the need for subprocess console parsing. It wraps the existing
|
|
BDFR core classes and provides a clean API with structured progress callbacks.
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from abc import ABC, abstractmethod
|
|
from collections.abc import Callable, Iterable
|
|
from datetime import datetime
|
|
from enum import Enum
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional, Union
|
|
|
|
import prawcore
|
|
|
|
from bdfr.configuration import Configuration
|
|
from bdfr.connector import RedditConnector
|
|
from bdfr.downloader import RedditDownloader
|
|
from bdfr.archiver import Archiver
|
|
from bdfr.cloner import RedditCloner
|
|
from bdfr import exceptions as errors
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def retry_reddit_api_call(api_call_func: Callable, max_retries: int = 5, base_wait_time: int = 60) -> Any:
|
|
"""
|
|
Retry wrapper for Reddit API calls that handles rate limiting (429 errors).
|
|
|
|
Args:
|
|
api_call_func: Function that makes the Reddit API call
|
|
max_retries: Maximum number of retry attempts (default: 5)
|
|
base_wait_time: Base wait time in seconds (default: 60)
|
|
|
|
Returns:
|
|
Result of the API call
|
|
|
|
Raises:
|
|
Exception: If all retry attempts are exhausted
|
|
"""
|
|
current_wait_time = base_wait_time
|
|
max_wait_time = base_wait_time * max_retries # Total max wait time
|
|
|
|
for attempt in range(max_retries + 1): # +1 for initial attempt
|
|
try:
|
|
logger.debug(f"Reddit API call attempt {attempt + 1}/{max_retries + 1}")
|
|
return api_call_func()
|
|
|
|
except Exception as e:
|
|
# Check if this is a rate limiting error
|
|
is_rate_limited = False
|
|
|
|
# Check for PRAW TooManyRequests exception
|
|
if isinstance(e, prawcore.exceptions.TooManyRequests):
|
|
is_rate_limited = True
|
|
logger.warning(f"Reddit API rate limited (TooManyRequests): {e}")
|
|
# Check for HTTP 429 in error message
|
|
elif "429" in str(e):
|
|
is_rate_limited = True
|
|
logger.warning(f"Reddit API rate limited (HTTP 429): {e}")
|
|
|
|
if not is_rate_limited:
|
|
# Not a rate limiting error, re-raise immediately
|
|
logger.error(f"Reddit API call failed with non-rate-limit error: {e}")
|
|
raise
|
|
|
|
# This is a rate limiting error
|
|
if attempt == max_retries:
|
|
# Last attempt failed, give up
|
|
logger.error(f"Reddit API rate limit retry attempts exhausted ({max_retries + 1} attempts)")
|
|
logger.error(f"Final error: {e}")
|
|
raise Exception(f"Reddit API rate limited after {max_retries + 1} attempts: {e}")
|
|
|
|
# Wait before retrying
|
|
logger.info(f"Reddit API rate limited, waiting {current_wait_time} seconds before retry {attempt + 1}/{max_retries}")
|
|
time.sleep(current_wait_time)
|
|
|
|
# Increase wait time for next attempt (exponential backoff)
|
|
current_wait_time = min(current_wait_time + base_wait_time, max_wait_time)
|
|
|
|
|
|
class DownloadType(Enum):
|
|
"""Types of downloads supported by BDFR"""
|
|
SUBREDDIT = "subreddit"
|
|
USER = "user"
|
|
MULTIREDDIT = "multireddit"
|
|
SUBMISSIONS = "submissions"
|
|
ARCHIVE = "archive"
|
|
CLONE = "clone"
|
|
|
|
|
|
class DownloadStatus(Enum):
|
|
"""Status of a download operation"""
|
|
QUEUED = "queued"
|
|
RUNNING = "running"
|
|
COMPLETED = "completed"
|
|
FAILED = "failed"
|
|
CANCELLED = "cancelled"
|
|
PAUSED = "paused"
|
|
|
|
|
|
class ProgressEvent:
|
|
"""Represents a progress event with structured data"""
|
|
|
|
def __init__(
|
|
self,
|
|
event_type: str,
|
|
download_id: str,
|
|
message: str,
|
|
progress: Optional[float] = None,
|
|
data: Optional[Dict[str, Any]] = None,
|
|
timestamp: Optional[datetime] = None
|
|
):
|
|
self.event_type = event_type # "progress", "status", "error", "completed"
|
|
self.download_id = download_id
|
|
self.message = message
|
|
self.progress = progress # 0-100 percentage
|
|
self.data = data or {}
|
|
self.timestamp = timestamp or datetime.now()
|
|
|
|
def to_dict(self) -> Dict[str, Any]:
|
|
"""Convert to dictionary for JSON serialization"""
|
|
return {
|
|
"event_type": self.event_type,
|
|
"download_id": self.download_id,
|
|
"message": self.message,
|
|
"progress": self.progress,
|
|
"data": self.data,
|
|
"timestamp": self.timestamp.isoformat()
|
|
}
|
|
|
|
|
|
class ProgressCallback(ABC):
|
|
"""Abstract base class for progress callbacks"""
|
|
|
|
@abstractmethod
|
|
async def on_progress(self, event: ProgressEvent):
|
|
"""Called when progress is made"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
async def on_error(self, event: ProgressEvent):
|
|
"""Called when an error occurs"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
async def on_completed(self, event: ProgressEvent):
|
|
"""Called when download is completed"""
|
|
pass
|
|
|
|
|
|
class LoggingCallback(ProgressCallback):
|
|
"""Default logging-based progress callback"""
|
|
|
|
def __init__(self, logger_name: str = __name__):
|
|
self.logger = logging.getLogger(logger_name)
|
|
|
|
async def on_progress(self, event: ProgressEvent):
|
|
"""Log progress events"""
|
|
if event.progress is not None:
|
|
self.logger.info(f"[{event.download_id}] {event.message} ({int(round(event.progress))}%)")
|
|
else:
|
|
self.logger.info(f"[{event.download_id}] {event.message}")
|
|
|
|
async def on_error(self, event: ProgressEvent):
|
|
"""Log error events"""
|
|
self.logger.error(f"[{event.download_id}] ERROR: {event.message}")
|
|
if event.data.get('exception'):
|
|
self.logger.exception(f"[{event.download_id}] Exception details", exc_info=event.data['exception'])
|
|
|
|
async def on_completed(self, event: ProgressEvent):
|
|
"""Log completion events"""
|
|
if event.event_type == "completed":
|
|
self.logger.info(f"[{event.download_id}] {event.message}")
|
|
else:
|
|
self.logger.info(f"[{event.download_id}] {event.message}")
|
|
|
|
|
|
class BDFRManager:
|
|
"""
|
|
Main API interface for BDFR operations.
|
|
|
|
This class provides a clean interface for the web application to interact
|
|
with BDFR functionality without needing subprocess calls.
|
|
"""
|
|
|
|
def __init__(self, download_directory: Optional[Union[str, Path]] = None, auth_token: Optional[str] = None):
|
|
"""
|
|
Initialize the BDFR Manager.
|
|
|
|
Args:
|
|
download_directory: Base directory for downloads. If None, uses current directory.
|
|
auth_token: Optional authentication token for Reddit API
|
|
"""
|
|
self.download_directory = Path(download_directory or ".").resolve()
|
|
self.download_directory.mkdir(exist_ok=True, parents=True)
|
|
self.auth_token = auth_token
|
|
|
|
# Active downloads tracking
|
|
self._active_downloads: Dict[str, Dict[str, Any]] = {}
|
|
self._download_threads: Dict[str, threading.Thread] = {}
|
|
self._callbacks: Dict[str, List[ProgressCallback]] = {}
|
|
|
|
# Setup logging first
|
|
self.logger = logging.getLogger(__name__)
|
|
|
|
# Default configuration
|
|
self._default_config = self._create_default_config()
|
|
|
|
# Override the log file path to avoid conflicts between multiple BDFR instances
|
|
if not self._default_config.log:
|
|
# Create a unique log file for this BDFR manager instance
|
|
log_dir = self.download_directory / "logs"
|
|
log_dir.mkdir(exist_ok=True)
|
|
unique_log_file = log_dir / f"bdfr_manager_{int(time.time())}_{id(self)}.log"
|
|
self._default_config.log = str(unique_log_file)
|
|
|
|
def _create_default_config(self) -> Configuration:
|
|
"""Create a default configuration for BDFR operations"""
|
|
config = Configuration()
|
|
config.directory = str(self.download_directory)
|
|
config.limit = 100 # Default limit
|
|
config.sort = "hot"
|
|
config.time = "all"
|
|
config.no_dupes = False
|
|
config.make_hard_links = False
|
|
config.verbose = 1
|
|
|
|
# Set authentication if token provided
|
|
if self.auth_token:
|
|
config.authenticate = True
|
|
self.logger.info(f"[DEBUG] Authentication enabled for download with token")
|
|
else:
|
|
config.authenticate = False
|
|
self.logger.info(f"[DEBUG] No authentication token provided, using anonymous access")
|
|
|
|
# Archiver defaults
|
|
config.format = "json"
|
|
config.all_comments = False
|
|
config.comment_context = False
|
|
|
|
return config
|
|
|
|
def create_download(
|
|
self,
|
|
download_type: DownloadType,
|
|
name: str,
|
|
config: Optional[Configuration] = None,
|
|
progress_callbacks: Optional[List[ProgressCallback]] = None
|
|
) -> str:
|
|
"""
|
|
Create a new download operation.
|
|
|
|
Args:
|
|
download_type: Type of download (subreddit, user, etc.)
|
|
name: Name of the target (subreddit name, username, etc.)
|
|
config: BDFR configuration. If None, uses defaults.
|
|
progress_callbacks: List of progress callbacks. If None, uses logging callback.
|
|
|
|
Returns:
|
|
Download ID for tracking the operation
|
|
"""
|
|
download_id = str(uuid.uuid4())
|
|
|
|
# Use provided config or default
|
|
if config is None:
|
|
# Create a new Configuration instance with default values
|
|
config = Configuration()
|
|
config.directory = str(self.download_directory)
|
|
config.limit = 100 # Default limit
|
|
config.sort = "hot"
|
|
config.time = "all"
|
|
config.no_dupes = False
|
|
config.make_hard_links = False
|
|
config.verbose = 1
|
|
config.format = "json"
|
|
config.all_comments = False
|
|
config.comment_context = False
|
|
|
|
# Create unique log file for this download
|
|
log_dir = self.download_directory / "logs"
|
|
log_dir.mkdir(exist_ok=True)
|
|
unique_log_file = log_dir / f"bdfr_{download_type.value}_{name}_{download_id}_{int(time.time())}.log"
|
|
config.log = str(unique_log_file)
|
|
|
|
# Set specific parameters based on download type
|
|
if download_type == DownloadType.SUBREDDIT:
|
|
config.subreddit = [name]
|
|
elif download_type == DownloadType.USER:
|
|
config.user = [name]
|
|
# Append username to directory path for user downloads
|
|
# This creates folder structure: downloads/username/subreddit/files
|
|
user_directory = self.download_directory / name
|
|
config.directory = str(user_directory)
|
|
elif download_type == DownloadType.MULTIREDDIT:
|
|
config.multireddit = [name]
|
|
|
|
# Setup callbacks
|
|
if progress_callbacks is None:
|
|
progress_callbacks = [LoggingCallback()]
|
|
|
|
self._callbacks[download_id] = progress_callbacks
|
|
|
|
# Initialize download tracking
|
|
self._active_downloads[download_id] = {
|
|
"id": download_id,
|
|
"type": download_type.value,
|
|
"name": name,
|
|
"status": DownloadStatus.QUEUED.value,
|
|
"progress": 0.0,
|
|
"start_time": datetime.now(),
|
|
"config": config,
|
|
"callbacks": progress_callbacks,
|
|
"items_processed": 0,
|
|
"items_found": 0,
|
|
"current_item": None,
|
|
"phase": "queued"
|
|
}
|
|
|
|
self.logger.info(f"Created download {download_id} for {download_type.value}: {name}")
|
|
return download_id
|
|
|
|
def start_download(self, download_id: str) -> bool:
|
|
"""
|
|
Start a download operation asynchronously.
|
|
|
|
Args:
|
|
download_id: ID of the download to start
|
|
|
|
Returns:
|
|
True if started successfully, False if download not found or already running
|
|
"""
|
|
if download_id not in self._active_downloads:
|
|
self.logger.error(f"Download {download_id} not found")
|
|
return False
|
|
|
|
download_info = self._active_downloads[download_id]
|
|
if download_info["status"] == DownloadStatus.RUNNING.value:
|
|
self.logger.warning(f"Download {download_id} is already running")
|
|
return False
|
|
|
|
# Update status to running
|
|
download_info["status"] = DownloadStatus.RUNNING.value
|
|
download_info["start_time"] = datetime.now()
|
|
|
|
# Create and start download thread
|
|
download_thread = threading.Thread(
|
|
target=self._run_download,
|
|
args=(download_id,),
|
|
daemon=True
|
|
)
|
|
self._download_threads[download_id] = download_thread
|
|
download_thread.start()
|
|
|
|
self.logger.info(f"Started download {download_id}")
|
|
return True
|
|
|
|
def _run_download(self, download_id: str):
|
|
"""Internal method to run a download in a separate thread"""
|
|
download_info = self._active_downloads[download_id]
|
|
config = download_info["config"]
|
|
callbacks = download_info["callbacks"]
|
|
downloader = None
|
|
|
|
try:
|
|
self.logger.info(f"[DEBUG] Starting download {download_id} for {download_info['type']}: {download_info['name']}")
|
|
self.logger.info(f"[DEBUG] Config: subreddit={getattr(config, 'subreddit', None)}, user={getattr(config, 'user', None)}, limit={getattr(config, 'limit', None)}")
|
|
|
|
# Create progress event for start
|
|
start_event = ProgressEvent(
|
|
"status",
|
|
download_id,
|
|
f"Starting {download_info['type']} download: {download_info['name']}",
|
|
0.0,
|
|
{"phase": "starting"}
|
|
)
|
|
|
|
# Notify callbacks
|
|
self.logger.info(f"[BDFR-API] Notifying {len(callbacks)} callbacks of progress event for download {download_id}")
|
|
asyncio.run(self._notify_callbacks(callbacks, "on_progress", start_event))
|
|
|
|
# Set up authentication token in config if provided
|
|
if self.auth_token:
|
|
# Store the token in the config for the connector to use
|
|
config.auth_token = self.auth_token
|
|
self.logger.info(f"[DEBUG] Set auth token in config")
|
|
|
|
# Ensure unique log file for this download to avoid conflicts
|
|
if not config.log:
|
|
# Create a unique log file for this download attempt
|
|
log_dir = self.download_directory / "logs"
|
|
log_dir.mkdir(exist_ok=True)
|
|
unique_log_file = log_dir / f"bdfr_download_{download_id}_{int(time.time())}.log"
|
|
config.log = str(unique_log_file)
|
|
|
|
# Ensure log directory exists before creating downloader
|
|
if config.log:
|
|
log_path = Path(config.log)
|
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Determine which BDFR class to use based on configuration
|
|
self.logger.info(f"[DEBUG] Determining downloader type - submitted={getattr(config, 'submitted', False)}, upvoted={getattr(config, 'upvoted', False)}, saved={getattr(config, 'saved', False)}, format={getattr(config, 'format', 'N/A')}")
|
|
|
|
try:
|
|
if config.submitted or config.upvoted or config.saved:
|
|
# User data download
|
|
self.logger.info(f"[DEBUG] Creating RedditDownloader for user data")
|
|
downloader = RedditDownloader(config, [])
|
|
download_info["operation_type"] = "user_download"
|
|
self.logger.info(f"[DEBUG] Using RedditDownloader for user data")
|
|
elif hasattr(config, 'format') and config.format in ["json", "xml", "yaml"]:
|
|
# Archive operation
|
|
self.logger.info(f"[DEBUG] Creating Archiver for archive operation")
|
|
downloader = Archiver(config, [])
|
|
download_info["operation_type"] = "archive"
|
|
self.logger.info(f"[DEBUG] Using Archiver for archive operation")
|
|
else:
|
|
# Standard download
|
|
self.logger.info(f"[DEBUG] Creating RedditDownloader for standard download")
|
|
downloader = RedditDownloader(config, [])
|
|
download_info["operation_type"] = "download"
|
|
self.logger.info(f"[DEBUG] Using RedditDownloader for standard download")
|
|
except Exception as e:
|
|
self.logger.error(f"[DEBUG] Failed to create downloader: {e}")
|
|
raise
|
|
|
|
# Run the actual download with progress tracking
|
|
self.logger.info(f"[DEBUG] Starting download_with_progress for {download_id}")
|
|
try:
|
|
self._download_with_progress(download_id, downloader, callbacks, config)
|
|
except Exception as e:
|
|
self.logger.error(f"[DEBUG] Exception in download_with_progress: {e}")
|
|
raise
|
|
|
|
# Mark as completed only if not already failed or cancelled
|
|
if download_info.get("status") not in [DownloadStatus.FAILED.value, DownloadStatus.CANCELLED.value]:
|
|
self.logger.info(f"[DEBUG] Marking download {download_id} as completed")
|
|
self.logger.info(f"[DEBUG] Final stats - items_processed: {download_info['items_processed']}, items_found: {download_info['items_found']}")
|
|
download_info["status"] = DownloadStatus.COMPLETED.value
|
|
download_info["progress"] = 100.0
|
|
download_info["end_time"] = datetime.now()
|
|
|
|
completed_event = ProgressEvent(
|
|
"completed",
|
|
download_id,
|
|
f"Download completed successfully",
|
|
100.0,
|
|
{
|
|
"items_processed": download_info["items_processed"],
|
|
"items_found": download_info["items_found"],
|
|
"operation_type": download_info.get("operation_type")
|
|
}
|
|
)
|
|
|
|
self.logger.info(f"[DEBUG] Sending completion callback for {download_id}")
|
|
self.logger.info(f"[BDFR-API] Sending completion event for download {download_id}")
|
|
asyncio.run(self._notify_callbacks(callbacks, "on_completed", completed_event))
|
|
else:
|
|
self.logger.info(f"[DEBUG] Skipping completion for {download_id} due to status={download_info.get('status')}")
|
|
|
|
except Exception as e:
|
|
import traceback
|
|
# Get full stack trace for debugging
|
|
stack_trace = traceback.format_exc()
|
|
|
|
# Determine if this failure was due to rate limiting (HTTP 429)
|
|
is_rate_limited = False
|
|
try:
|
|
import prawcore
|
|
TooManyRequests = getattr(prawcore.exceptions, "TooManyRequests", None)
|
|
if TooManyRequests is not None and isinstance(e, TooManyRequests):
|
|
is_rate_limited = True
|
|
self.logger.info(f"[DEBUG] Exception is TooManyRequests for download {download_id}")
|
|
except Exception:
|
|
pass
|
|
if "429" in str(e):
|
|
is_rate_limited = True
|
|
self.logger.info(f"[DEBUG] Error message contains '429' for download {download_id}: {str(e)}")
|
|
|
|
self.logger.error(f"[DEBUG] Exception caught in _run_download for {download_id}: {e}")
|
|
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
|
|
|
|
# Mark as failed
|
|
download_info["status"] = DownloadStatus.FAILED.value
|
|
download_info["error"] = str(e)
|
|
download_info["stack_trace"] = stack_trace
|
|
download_info["end_time"] = datetime.now()
|
|
|
|
error_event = ProgressEvent(
|
|
"error",
|
|
download_id,
|
|
f"Download failed: {str(e)}",
|
|
None,
|
|
{"exception": str(e), "stack_trace": stack_trace, "phase": ("rate_limited" if is_rate_limited else "failed")}
|
|
)
|
|
|
|
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
|
|
|
|
finally:
|
|
# Clean up thread reference
|
|
if download_id in self._download_threads:
|
|
del self._download_threads[download_id]
|
|
|
|
# Clean up logging handlers to prevent file locking issues
|
|
if downloader is not None:
|
|
try:
|
|
# Close any logging handlers that might be holding file locks
|
|
logger = logging.getLogger()
|
|
for handler in logger.handlers[:]:
|
|
if hasattr(handler, 'close') and hasattr(handler, 'baseFilename'):
|
|
# Check if this handler is for our download's log file
|
|
if hasattr(config, 'log') and config.log:
|
|
handler_path = getattr(handler, 'baseFilename', '')
|
|
if handler_path == config.log or handler_path.startswith(str(self.download_directory)):
|
|
try:
|
|
handler.close()
|
|
logger.removeHandler(handler)
|
|
self.logger.info(f"[DEBUG] Closed logging handler for {handler_path}")
|
|
except Exception as cleanup_error:
|
|
self.logger.warning(f"[DEBUG] Error closing handler {handler_path}: {cleanup_error}")
|
|
except Exception as cleanup_error:
|
|
self.logger.warning(f"[DEBUG] Error during logging cleanup: {cleanup_error}")
|
|
|
|
def _download_with_progress(self, download_id: str, downloader, callbacks: List[ProgressCallback], config):
|
|
"""Run download with progress tracking and callbacks"""
|
|
download_info = self._active_downloads[download_id]
|
|
|
|
# Create a progress-tracking wrapper for the downloader
|
|
original_download = downloader.download
|
|
|
|
def progress_download():
|
|
"""Wrapper for download method that sends progress updates"""
|
|
try:
|
|
# Initialize submission variable before any potential exceptions
|
|
submission = None
|
|
|
|
# Update phase to fetching
|
|
download_info["phase"] = "fetching_submissions"
|
|
progress_event = ProgressEvent(
|
|
"progress",
|
|
download_id,
|
|
"Fetching submissions...",
|
|
10.0,
|
|
{"phase": "fetching_submissions"}
|
|
)
|
|
self.logger.info(f"[BDFR-API] Sending progress update for download {download_id}")
|
|
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
|
|
|
|
# For RedditDownloader, we need to intercept the download process
|
|
if hasattr(downloader, 'reddit_lists') and hasattr(downloader, '_download_submission'):
|
|
processed_submissions = 0
|
|
|
|
self.logger.info(f"[DEBUG] RedditDownloader has reddit_lists and _download_submission methods")
|
|
|
|
# Get the limit from config - this is our target/goal for items_found
|
|
# items_found represents "how many items are available to download (up to limit)"
|
|
# items_processed represents "how many we've actually processed so far"
|
|
limit = getattr(config, 'limit', 100)
|
|
download_info["items_found"] = limit
|
|
self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}")
|
|
|
|
# Process submissions with progress tracking
|
|
self.logger.info(f"[DEBUG] Starting submission processing loop...")
|
|
actual_count = 0 # Track actual submissions found for final adjustment
|
|
|
|
for generator in downloader.reddit_lists:
|
|
self.logger.info(f"[DEBUG] Processing generator...")
|
|
submission_count = 0
|
|
for submission in generator:
|
|
# Ensure submission is defined for the entire loop
|
|
if submission is None:
|
|
continue
|
|
submission_count += 1
|
|
actual_count += 1
|
|
|
|
if download_info["status"] == DownloadStatus.CANCELLED.value:
|
|
self.logger.info(f"Download {download_id} was cancelled")
|
|
return
|
|
|
|
try:
|
|
self.logger.info(f"[DEBUG] Processing submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}")
|
|
|
|
# Update current item being processed
|
|
download_info["current_item"] = submission.id
|
|
download_info["current_item_type"] = "submission"
|
|
download_info["current_subreddit"] = submission.subreddit.display_name
|
|
|
|
# Send progress update for this submission
|
|
processed_submissions += 1
|
|
# Calculate progress based on processed vs expected (limit), but cap at 90%
|
|
# This gives smooth progress even when we don't know the exact total yet
|
|
progress_percent = 20.0 + (processed_submissions / max(limit, 1)) * 70.0 # 20-90% range
|
|
|
|
progress_event = ProgressEvent(
|
|
"progress",
|
|
download_id,
|
|
f"Processing submission: {submission.id}",
|
|
min(progress_percent, 90.0),
|
|
{
|
|
"phase": "downloading_submission",
|
|
"current_item": submission.id,
|
|
"current_item_type": "submission",
|
|
"current_subreddit": submission.subreddit.display_name,
|
|
"items_processed": processed_submissions,
|
|
"items_found": download_info["items_found"] # Keep showing the limit as target
|
|
}
|
|
)
|
|
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
|
|
|
|
# Process this submission with retry mechanism for rate limiting
|
|
self.logger.info(f"[DEBUG] Calling _download_submission for {submission.id}")
|
|
try:
|
|
downloader._download_submission(submission)
|
|
self.logger.info(f"[DEBUG] Completed _download_submission for {submission.id}")
|
|
except Exception as e:
|
|
# Check if this is a rate limiting error that should be retried
|
|
error_msg = str(e)
|
|
if "429" in error_msg:
|
|
self.logger.warning(f"[DEBUG] Rate limited while processing submission {submission.id}: {e}")
|
|
self.logger.warning(f"[DEBUG] Will continue with next submission instead of failing entire download")
|
|
continue
|
|
else:
|
|
# Not a rate limiting error, re-raise
|
|
raise
|
|
|
|
# Update processed count
|
|
download_info["items_processed"] = processed_submissions
|
|
|
|
except Exception as e:
|
|
import traceback
|
|
error_msg = str(e)
|
|
submission_id = submission.id if submission is not None else "unknown"
|
|
|
|
# Get full stack trace for debugging
|
|
stack_trace = traceback.format_exc()
|
|
|
|
if "429" in error_msg:
|
|
self.logger.error(f"[DEBUG] Rate limited while processing submission {submission_id}: {e}")
|
|
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
|
|
# Mark download as failed due to rate limiting
|
|
download_info["status"] = DownloadStatus.FAILED.value
|
|
download_info["error"] = f"Rate limited by Reddit API: {e}"
|
|
download_info["end_time"] = datetime.now()
|
|
|
|
error_event = ProgressEvent(
|
|
"error",
|
|
download_id,
|
|
f"Rate limited by Reddit API: {e}",
|
|
None,
|
|
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"}
|
|
)
|
|
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
|
|
return
|
|
else:
|
|
self.logger.error(f"[DEBUG] Error processing submission {submission_id}: {e}")
|
|
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
|
|
continue
|
|
|
|
self.logger.info(f"[DEBUG] Generator processed {submission_count} submissions")
|
|
|
|
# Final progress update
|
|
# If we found fewer items than the limit, update items_found to reflect reality
|
|
if actual_count < limit:
|
|
download_info["items_found"] = actual_count
|
|
self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}")
|
|
|
|
self.logger.info(f"[DEBUG] Final progress update - processed {processed_submissions} out of {download_info['items_found']} found")
|
|
|
|
progress_event = ProgressEvent(
|
|
"progress",
|
|
download_id,
|
|
"Download completed",
|
|
100.0,
|
|
{
|
|
"phase": "completed",
|
|
"items_processed": processed_submissions,
|
|
"items_found": download_info["items_found"]
|
|
}
|
|
)
|
|
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
|
|
|
|
elif hasattr(downloader, 'reddit_lists') and hasattr(downloader, 'write_entry'):
|
|
# For Archiver, we need to intercept the archiving process
|
|
processed_items = 0
|
|
|
|
self.logger.info(f"[DEBUG] Archiver detected with reddit_lists and write_entry methods")
|
|
|
|
# Get the limit from config
|
|
limit = getattr(config, 'limit', 100)
|
|
download_info["items_found"] = limit
|
|
self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}")
|
|
|
|
# Process items with progress tracking
|
|
self.logger.info(f"[DEBUG] Starting archiving loop...")
|
|
actual_count = 0
|
|
|
|
for generator in downloader.reddit_lists:
|
|
self.logger.info(f"[DEBUG] Processing generator for archiving...")
|
|
item_count = 0
|
|
for item in generator:
|
|
if item is None:
|
|
continue
|
|
item_count += 1
|
|
actual_count += 1
|
|
|
|
if download_info["status"] == DownloadStatus.CANCELLED.value:
|
|
self.logger.info(f"Download {download_id} was cancelled")
|
|
return
|
|
|
|
try:
|
|
# Determine item type
|
|
import praw.models
|
|
if isinstance(item, praw.models.Submission):
|
|
item_id = item.id
|
|
item_type = "submission"
|
|
subreddit_name = item.subreddit.display_name
|
|
elif isinstance(item, praw.models.Comment):
|
|
item_id = item.id
|
|
item_type = "comment"
|
|
subreddit_name = item.subreddit.display_name if hasattr(item, 'subreddit') else "unknown"
|
|
else:
|
|
item_id = str(item)
|
|
item_type = "item"
|
|
subreddit_name = "unknown"
|
|
|
|
self.logger.info(f"[DEBUG] Archiving {item_type} {item_count}: {item_id} from r/{subreddit_name}")
|
|
|
|
# Update current item being processed
|
|
download_info["current_item"] = item_id
|
|
download_info["current_item_type"] = item_type
|
|
download_info["current_subreddit"] = subreddit_name
|
|
|
|
# Send progress update for this item
|
|
processed_items += 1
|
|
progress_percent = 20.0 + (processed_items / max(limit, 1)) * 70.0
|
|
|
|
progress_event = ProgressEvent(
|
|
"progress",
|
|
download_id,
|
|
f"Archiving {item_type}: {item_id}",
|
|
min(progress_percent, 90.0),
|
|
{
|
|
"phase": "archiving_item",
|
|
"current_item": item_id,
|
|
"current_item_type": item_type,
|
|
"current_subreddit": subreddit_name,
|
|
"items_processed": processed_items,
|
|
"items_found": download_info["items_found"]
|
|
}
|
|
)
|
|
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
|
|
|
|
# Archive this item with retry mechanism for rate limiting
|
|
self.logger.info(f"[DEBUG] Calling write_entry for {item_id}")
|
|
try:
|
|
downloader.write_entry(item)
|
|
self.logger.info(f"[DEBUG] Completed write_entry for {item_id}")
|
|
except Exception as e:
|
|
# Check if this is a rate limiting error that should be retried
|
|
error_msg = str(e)
|
|
if "429" in error_msg:
|
|
self.logger.warning(f"[DEBUG] Rate limited while archiving item {item_id}: {e}")
|
|
self.logger.warning(f"[DEBUG] Will continue with next item instead of failing entire download")
|
|
continue
|
|
else:
|
|
# Not a rate limiting error, re-raise
|
|
raise
|
|
|
|
# Update processed count
|
|
download_info["items_processed"] = processed_items
|
|
|
|
except Exception as e:
|
|
import traceback
|
|
error_msg = str(e)
|
|
item_id = item_id if 'item_id' in locals() else "unknown"
|
|
stack_trace = traceback.format_exc()
|
|
|
|
if "429" in error_msg:
|
|
self.logger.error(f"[DEBUG] Rate limited while archiving item {item_id}: {e}")
|
|
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
|
|
download_info["status"] = DownloadStatus.FAILED.value
|
|
download_info["error"] = f"Rate limited by Reddit API: {e}"
|
|
download_info["end_time"] = datetime.now()
|
|
|
|
error_event = ProgressEvent(
|
|
"error",
|
|
download_id,
|
|
f"Rate limited by Reddit API: {e}",
|
|
None,
|
|
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"}
|
|
)
|
|
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
|
|
return
|
|
else:
|
|
self.logger.error(f"[DEBUG] Error archiving item {item_id}: {e}")
|
|
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
|
|
continue
|
|
|
|
self.logger.info(f"[DEBUG] Generator archived {item_count} items")
|
|
|
|
# Final progress update
|
|
if actual_count < limit:
|
|
download_info["items_found"] = actual_count
|
|
self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}")
|
|
|
|
self.logger.info(f"[DEBUG] Final progress update - archived {processed_items} out of {download_info['items_found']} found")
|
|
|
|
progress_event = ProgressEvent(
|
|
"progress",
|
|
download_id,
|
|
"Archiving completed",
|
|
100.0,
|
|
{
|
|
"phase": "completed",
|
|
"items_processed": processed_items,
|
|
"items_found": download_info["items_found"]
|
|
}
|
|
)
|
|
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
|
|
|
|
elif hasattr(downloader, 'reddit_lists') and hasattr(downloader, '_download_submission') and hasattr(downloader, 'write_entry'):
|
|
# For RedditCloner (which has both download and archive capabilities)
|
|
processed_items = 0
|
|
|
|
self.logger.info(f"[DEBUG] RedditCloner detected with reddit_lists, _download_submission, and write_entry methods")
|
|
|
|
# Get the limit from config
|
|
limit = getattr(config, 'limit', 100)
|
|
download_info["items_found"] = limit
|
|
self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}")
|
|
|
|
# Process submissions with progress tracking (both download and archive)
|
|
self.logger.info(f"[DEBUG] Starting cloning loop...")
|
|
actual_count = 0
|
|
|
|
for generator in downloader.reddit_lists:
|
|
self.logger.info(f"[DEBUG] Processing generator for cloning...")
|
|
submission_count = 0
|
|
for submission in generator:
|
|
if submission is None:
|
|
continue
|
|
submission_count += 1
|
|
actual_count += 1
|
|
|
|
if download_info["status"] == DownloadStatus.CANCELLED.value:
|
|
self.logger.info(f"Download {download_id} was cancelled")
|
|
return
|
|
|
|
try:
|
|
self.logger.info(f"[DEBUG] Cloning submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}")
|
|
|
|
# Update current item being processed
|
|
download_info["current_item"] = submission.id
|
|
download_info["current_item_type"] = "submission"
|
|
download_info["current_subreddit"] = submission.subreddit.display_name
|
|
|
|
# Send progress update for this submission
|
|
processed_items += 1
|
|
progress_percent = 20.0 + (processed_items / max(limit, 1)) * 70.0
|
|
|
|
progress_event = ProgressEvent(
|
|
"progress",
|
|
download_id,
|
|
f"Cloning submission: {submission.id}",
|
|
min(progress_percent, 90.0),
|
|
{
|
|
"phase": "cloning_submission",
|
|
"current_item": submission.id,
|
|
"current_item_type": "submission",
|
|
"current_subreddit": submission.subreddit.display_name,
|
|
"items_processed": processed_items,
|
|
"items_found": download_info["items_found"]
|
|
}
|
|
)
|
|
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
|
|
|
|
# Clone this submission (download + archive) with retry mechanism for rate limiting
|
|
self.logger.info(f"[DEBUG] Calling _download_submission for {submission.id}")
|
|
try:
|
|
downloader._download_submission(submission)
|
|
self.logger.info(f"[DEBUG] Calling write_entry for {submission.id}")
|
|
downloader.write_entry(submission)
|
|
self.logger.info(f"[DEBUG] Completed cloning for {submission.id}")
|
|
except Exception as e:
|
|
# Check if this is a rate limiting error that should be retried
|
|
error_msg = str(e)
|
|
if "429" in error_msg:
|
|
self.logger.warning(f"[DEBUG] Rate limited while cloning submission {submission.id}: {e}")
|
|
self.logger.warning(f"[DEBUG] Will continue with next submission instead of failing entire download")
|
|
continue
|
|
else:
|
|
# Not a rate limiting error, re-raise
|
|
raise
|
|
|
|
# Update processed count
|
|
download_info["items_processed"] = processed_items
|
|
|
|
except Exception as e:
|
|
import traceback
|
|
error_msg = str(e)
|
|
submission_id = submission.id if submission is not None else "unknown"
|
|
stack_trace = traceback.format_exc()
|
|
|
|
if "429" in error_msg:
|
|
self.logger.error(f"[DEBUG] Rate limited while cloning submission {submission_id}: {e}")
|
|
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
|
|
self.logger.info(f"[DEBUG] Error message contains '429': {error_msg}")
|
|
download_info["status"] = DownloadStatus.FAILED.value
|
|
download_info["error"] = f"Rate limited by Reddit API: {e}"
|
|
download_info["end_time"] = datetime.now()
|
|
|
|
error_event = ProgressEvent(
|
|
"error",
|
|
download_id,
|
|
f"Rate limited by Reddit API: {e}",
|
|
None,
|
|
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"}
|
|
)
|
|
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
|
|
return
|
|
else:
|
|
self.logger.error(f"[DEBUG] Error cloning submission {submission_id}: {e}")
|
|
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
|
|
continue
|
|
|
|
self.logger.info(f"[DEBUG] Generator cloned {submission_count} submissions")
|
|
|
|
# Final progress update
|
|
if actual_count < limit:
|
|
download_info["items_found"] = actual_count
|
|
self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}")
|
|
|
|
self.logger.info(f"[DEBUG] Final progress update - cloned {processed_items} out of {download_info['items_found']} found")
|
|
|
|
progress_event = ProgressEvent(
|
|
"progress",
|
|
download_id,
|
|
"Cloning completed",
|
|
100.0,
|
|
{
|
|
"phase": "completed",
|
|
"items_processed": processed_items,
|
|
"items_found": download_info["items_found"]
|
|
}
|
|
)
|
|
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
|
|
|
|
else:
|
|
# For other downloaders, just run normally
|
|
self.logger.info(f"[DEBUG] Using fallback downloader (no specific handler)")
|
|
original_download()
|
|
|
|
except Exception as e:
|
|
import traceback
|
|
stack_trace = traceback.format_exc()
|
|
self.logger.error(f"Error in progress download for {download_id}: {e}")
|
|
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
|
|
self.logger.info(f"[DEBUG] Exception caught in progress_download, raising for {download_id}")
|
|
raise
|
|
|
|
# Set up authentication if token provided
|
|
if hasattr(config, 'auth_token') and config.auth_token:
|
|
self.logger.info(f"[DEBUG] Setting up authenticated Reddit instance")
|
|
# Monkey patch the create_reddit_instance method to use our token
|
|
original_create_reddit_instance = downloader.create_reddit_instance
|
|
auth_token = config.auth_token # Capture token in closure
|
|
|
|
def authenticated_create_reddit_instance():
|
|
try:
|
|
self.logger.info(f"[DEBUG] Creating authenticated Reddit instance")
|
|
# Call original method first to set up basic config
|
|
original_create_reddit_instance()
|
|
|
|
# Override the reddit_instance with authenticated version
|
|
import praw
|
|
downloader.reddit_instance = praw.Reddit(
|
|
client_id=downloader.cfg_parser.get("DEFAULT", "client_id"),
|
|
client_secret=downloader.cfg_parser.get("DEFAULT", "client_secret"),
|
|
user_agent=getattr(downloader, 'user_agent', 'BDFR-Web-Interface/1.0'),
|
|
token=auth_token
|
|
)
|
|
downloader.authenticated = True
|
|
self.logger.info(f"[DEBUG] Created authenticated Reddit instance successfully")
|
|
except Exception as e:
|
|
self.logger.error(f"[DEBUG] Failed to create authenticated Reddit instance: {e}")
|
|
raise
|
|
|
|
downloader.create_reddit_instance = authenticated_create_reddit_instance
|
|
|
|
# Replace the download method and run it
|
|
downloader.download = progress_download
|
|
downloader.download()
|
|
|
|
async def _notify_callbacks(self, callbacks: List[ProgressCallback], method_name: str, event: ProgressEvent):
|
|
"""Notify all callbacks of an event"""
|
|
self.logger.info(f"[BDFR-API] Notifying {len(callbacks)} callbacks of {method_name} event for download {event.download_id}")
|
|
for callback in callbacks:
|
|
try:
|
|
logger.info(f"[BDFR-API] Invoking {method_name} on callback {callback.__class__.__name__} for download {event.download_id}")
|
|
method = getattr(callback, method_name)
|
|
if asyncio.iscoroutinefunction(method):
|
|
await method(event)
|
|
else:
|
|
# Run sync method in thread pool
|
|
loop = asyncio.get_event_loop()
|
|
await loop.run_in_executor(None, lambda: asyncio.run(method(event)))
|
|
logger.info(f"[BDFR-API] Successfully invoked {method_name} on callback {callback.__class__.__name__}")
|
|
except Exception as e:
|
|
self.logger.error(f"[BDFR-API] Error in callback {callback.__class__.__name__}: {e}")
|
|
|
|
def get_download_status(self, download_id: str) -> Optional[Dict[str, Any]]:
|
|
"""
|
|
Get the current status of a download.
|
|
|
|
Args:
|
|
download_id: ID of the download to check
|
|
|
|
Returns:
|
|
Download status information or None if not found
|
|
"""
|
|
if download_id not in self._active_downloads:
|
|
return None
|
|
|
|
download_info = dict(self._active_downloads[download_id])
|
|
|
|
# Add thread status if running
|
|
if download_id in self._download_threads:
|
|
thread = self._download_threads[download_id]
|
|
download_info["thread_alive"] = thread.is_alive()
|
|
|
|
return download_info
|
|
|
|
def cancel_download(self, download_id: str) -> bool:
|
|
"""
|
|
Cancel a running download.
|
|
|
|
Args:
|
|
download_id: ID of the download to cancel
|
|
|
|
Returns:
|
|
True if cancelled successfully, False if download not found or not running
|
|
"""
|
|
if download_id not in self._active_downloads:
|
|
return False
|
|
|
|
download_info = self._active_downloads[download_id]
|
|
|
|
if download_info["status"] not in [DownloadStatus.RUNNING.value, DownloadStatus.QUEUED.value]:
|
|
return False
|
|
|
|
# Mark as cancelled
|
|
download_info["status"] = DownloadStatus.CANCELLED.value
|
|
download_info["end_time"] = datetime.now()
|
|
|
|
# Try to stop the thread (this is a best effort)
|
|
if download_id in self._download_threads:
|
|
thread = self._download_threads[download_id]
|
|
# Note: We can't actually kill the thread safely, but we can mark it as cancelled
|
|
# The BDFR instance would need to check for cancellation periodically
|
|
|
|
self.logger.info(f"Cancelled download {download_id}")
|
|
return True
|
|
|
|
def list_downloads(self) -> Dict[str, Dict[str, Any]]:
|
|
"""
|
|
List all downloads (active and recently completed).
|
|
|
|
Returns:
|
|
Dictionary of download information keyed by download ID
|
|
"""
|
|
return {
|
|
download_id: self.get_download_status(download_id)
|
|
for download_id in self._active_downloads
|
|
}
|
|
|
|
def cleanup_completed(self, max_age_seconds: int = 3600) -> int:
|
|
"""
|
|
Clean up completed/failed downloads older than specified age.
|
|
|
|
Args:
|
|
max_age_seconds: Maximum age in seconds for completed downloads to keep
|
|
|
|
Returns:
|
|
Number of downloads cleaned up
|
|
"""
|
|
current_time = datetime.now()
|
|
to_remove = []
|
|
|
|
for download_id, download_info in self._active_downloads.items():
|
|
if download_info["status"] in [DownloadStatus.COMPLETED.value, DownloadStatus.FAILED.value, DownloadStatus.CANCELLED.value]:
|
|
end_time = download_info.get("end_time")
|
|
if end_time and (current_time - end_time).total_seconds() > max_age_seconds:
|
|
to_remove.append(download_id)
|
|
|
|
for download_id in to_remove:
|
|
del self._active_downloads[download_id]
|
|
if download_id in self._callbacks:
|
|
del self._callbacks[download_id]
|
|
|
|
if to_remove:
|
|
self.logger.info(f"Cleaned up {len(to_remove)} old downloads: {to_remove}")
|
|
|
|
return len(to_remove)
|
|
|
|
# Convenience methods for common operations
|
|
|
|
def download_subreddit(
|
|
self,
|
|
subreddit_name: str,
|
|
limit: Optional[int] = None,
|
|
sort: str = "hot",
|
|
time_filter: str = "all",
|
|
no_dupes: bool = False,
|
|
simple_check: bool = False,
|
|
progress_callbacks: Optional[List[ProgressCallback]] = None
|
|
) -> str:
|
|
"""
|
|
Download content from a subreddit.
|
|
|
|
Args:
|
|
subreddit_name: Name of the subreddit (without r/)
|
|
limit: Maximum number of posts to download
|
|
sort: Sort method (hot, top, new, controversial, rising)
|
|
time_filter: Time filter (all, hour, day, week, month, year)
|
|
no_dupes: Whether to avoid duplicate downloads
|
|
simple_check: Whether to use simple URL-based duplicate checking
|
|
progress_callbacks: Optional progress callbacks
|
|
|
|
Returns:
|
|
Download ID for tracking
|
|
"""
|
|
# Create a new Configuration instance with default values
|
|
config = Configuration()
|
|
config.directory = str(self.download_directory)
|
|
config.limit = 100 # Default limit
|
|
config.sort = "hot"
|
|
config.time = "all"
|
|
config.no_dupes = False
|
|
config.make_hard_links = False
|
|
config.verbose = 1
|
|
config.format = "json"
|
|
config.all_comments = False
|
|
config.comment_context = False
|
|
|
|
# Create unique log file for this download
|
|
log_dir = self.download_directory / "logs"
|
|
log_dir.mkdir(exist_ok=True)
|
|
unique_log_file = log_dir / f"bdfr_subreddit_{subreddit_name}_{int(time.time())}.log"
|
|
config.log = str(unique_log_file)
|
|
|
|
config.subreddit = [subreddit_name]
|
|
if limit:
|
|
config.limit = limit
|
|
config.sort = sort
|
|
config.time = time_filter
|
|
config.no_dupes = no_dupes
|
|
config.simple_check = simple_check
|
|
|
|
download_id = self.create_download(
|
|
DownloadType.SUBREDDIT,
|
|
subreddit_name,
|
|
config,
|
|
progress_callbacks
|
|
)
|
|
|
|
self.start_download(download_id)
|
|
return download_id
|
|
|
|
def download_user(
|
|
self,
|
|
username: str,
|
|
limit: Optional[int] = None,
|
|
submitted: bool = True,
|
|
upvoted: bool = False,
|
|
saved: bool = False,
|
|
no_dupes: bool = False,
|
|
simple_check: bool = False,
|
|
progress_callbacks: Optional[List[ProgressCallback]] = None
|
|
) -> str:
|
|
"""
|
|
Download content from a user.
|
|
|
|
Args:
|
|
username: Reddit username
|
|
limit: Maximum number of posts to download
|
|
submitted: Download user's submitted posts
|
|
upvoted: Download user's upvoted posts (requires authentication)
|
|
saved: Download user's saved posts (requires authentication)
|
|
no_dupes: Whether to avoid duplicate downloads
|
|
progress_callbacks: Optional progress callbacks
|
|
|
|
Returns:
|
|
Download ID for tracking
|
|
"""
|
|
# Create a new Configuration instance with default values
|
|
config = Configuration()
|
|
# Append username to directory path for user downloads
|
|
# This creates folder structure: downloads/username/subreddit/files
|
|
user_directory = self.download_directory / username
|
|
config.directory = str(user_directory)
|
|
config.limit = 100 # Default limit
|
|
config.sort = "hot"
|
|
config.time = "all"
|
|
config.no_dupes = False
|
|
config.make_hard_links = False
|
|
config.verbose = 1
|
|
config.format = "json"
|
|
config.all_comments = False
|
|
config.comment_context = False
|
|
|
|
# Create unique log file for this download
|
|
log_dir = self.download_directory / "logs"
|
|
log_dir.mkdir(exist_ok=True)
|
|
unique_log_file = log_dir / f"bdfr_user_{username}_{int(time.time())}.log"
|
|
config.log = str(unique_log_file)
|
|
|
|
config.user = [username]
|
|
if limit:
|
|
config.limit = limit
|
|
config.submitted = submitted
|
|
config.upvoted = upvoted
|
|
config.saved = saved
|
|
config.no_dupes = no_dupes
|
|
config.simple_check = simple_check
|
|
|
|
download_id = self.create_download(
|
|
DownloadType.USER,
|
|
username,
|
|
config,
|
|
progress_callbacks
|
|
)
|
|
|
|
self.start_download(download_id)
|
|
return download_id
|
|
|
|
def archive_subreddit(
|
|
self,
|
|
subreddit_name: str,
|
|
format_type: str = "json",
|
|
limit: Optional[int] = None,
|
|
simple_check: bool = False,
|
|
progress_callbacks: Optional[List[ProgressCallback]] = None
|
|
) -> str:
|
|
"""
|
|
Archive subreddit data (metadata only, no downloads).
|
|
|
|
Args:
|
|
subreddit_name: Name of the subreddit
|
|
format_type: Archive format (json, xml, yaml)
|
|
limit: Maximum number of posts to archive
|
|
progress_callbacks: Optional progress callbacks
|
|
|
|
Returns:
|
|
Download ID for tracking
|
|
"""
|
|
# Create a new Configuration instance with default values
|
|
config = Configuration()
|
|
config.directory = str(self.download_directory)
|
|
config.limit = 100 # Default limit
|
|
config.sort = "hot"
|
|
config.time = "all"
|
|
config.no_dupes = False
|
|
config.make_hard_links = False
|
|
config.verbose = 1
|
|
config.format = "json"
|
|
config.all_comments = False
|
|
config.comment_context = False
|
|
|
|
# Create unique log file for this download
|
|
log_dir = self.download_directory / "logs"
|
|
log_dir.mkdir(exist_ok=True)
|
|
unique_log_file = log_dir / f"bdfr_archive_{subreddit_name}_{int(time.time())}.log"
|
|
config.log = str(unique_log_file)
|
|
|
|
config.subreddit = [subreddit_name]
|
|
if limit:
|
|
config.limit = limit
|
|
config.format = format_type
|
|
|
|
download_id = self.create_download(
|
|
DownloadType.ARCHIVE,
|
|
subreddit_name,
|
|
config,
|
|
progress_callbacks
|
|
)
|
|
|
|
self.start_download(download_id)
|
|
return download_id
|
|
|
|
def clone_subreddit(
|
|
self,
|
|
subreddit_name: str,
|
|
limit: Optional[int] = None,
|
|
format_type: str = "json",
|
|
no_dupes: bool = False,
|
|
simple_check: bool = False,
|
|
progress_callbacks: Optional[List[ProgressCallback]] = None
|
|
) -> str:
|
|
"""
|
|
Clone subreddit (both download and archive).
|
|
|
|
Args:
|
|
subreddit_name: Name of the subreddit
|
|
limit: Maximum number of posts to process
|
|
format_type: Archive format for metadata
|
|
no_dupes: Whether to avoid duplicate downloads
|
|
progress_callbacks: Optional progress callbacks
|
|
|
|
Returns:
|
|
Download ID for tracking
|
|
"""
|
|
# Create a new Configuration instance with default values
|
|
config = Configuration()
|
|
config.directory = str(self.download_directory)
|
|
config.limit = 100 # Default limit
|
|
config.sort = "hot"
|
|
config.time = "all"
|
|
config.no_dupes = False
|
|
config.make_hard_links = False
|
|
config.verbose = 1
|
|
config.format = "json"
|
|
config.all_comments = False
|
|
config.comment_context = False
|
|
|
|
# Create unique log file for this download
|
|
log_dir = self.download_directory / "logs"
|
|
log_dir.mkdir(exist_ok=True)
|
|
unique_log_file = log_dir / f"bdfr_clone_{subreddit_name}_{int(time.time())}.log"
|
|
config.log = str(unique_log_file)
|
|
|
|
config.subreddit = [subreddit_name]
|
|
if limit:
|
|
config.limit = limit
|
|
config.format = format_type
|
|
config.no_dupes = no_dupes
|
|
|
|
download_id = self.create_download(
|
|
DownloadType.CLONE,
|
|
subreddit_name,
|
|
config,
|
|
progress_callbacks
|
|
)
|
|
|
|
self.start_download(download_id)
|
|
return download_id
|
|
|
|
|
|
# Global instance for easy access
|
|
_default_manager: Optional[BDFRManager] = None
|
|
|
|
|
|
def get_bdfr_manager(download_directory: Optional[Union[str, Path]] = None) -> BDFRManager:
|
|
"""
|
|
Get the default BDFR manager instance.
|
|
|
|
Args:
|
|
download_directory: Base directory for downloads
|
|
|
|
Returns:
|
|
BDFRManager instance
|
|
"""
|
|
global _default_manager
|
|
if _default_manager is None:
|
|
_default_manager = BDFRManager(download_directory)
|
|
return _default_manager
|
|
|
|
|
|
def set_bdfr_manager(manager: BDFRManager):
|
|
"""
|
|
Set the default BDFR manager instance.
|
|
|
|
Args:
|
|
manager: BDFRManager instance to use as default
|
|
"""
|
|
global _default_manager
|
|
_default_manager = manager |