From 9e61d18bf6ded0c024b4b3e074333e0526df07c9 Mon Sep 17 00:00:00 2001 From: ModerateWinGuy Date: Thu, 9 Oct 2025 17:12:55 +1300 Subject: [PATCH] feat(UI): initial working frontend UI --- bdfr/API_README.md | 430 +++++++ bdfr/api.py | 1290 +++++++++++++++++++++ bdfr/archiver.py | 6 +- bdfr/cloner.py | 6 +- bdfr/connector.py | 12 + bdfr/downloader.py | 2 +- bdfr/examples/api_usage.py | 409 +++++++ tests/test_file_locking_fix.py | 148 +++ tests/test_user_folder_structure.py | 161 +++ web_interface/.dockerignore | 35 + web_interface/.env.example | 17 + web_interface/Dockerfile | 35 + web_interface/README.md | 202 ++++ web_interface/STARTUP.md | 177 +++ web_interface/app/auth.py | 311 +++++ web_interface/app/main.py | 960 +++++++++++++++ web_interface/requirements.txt | 11 + web_interface/setup_oauth.py | 123 ++ web_interface/start.bat | 105 ++ web_interface/start.py | 119 ++ web_interface/start.sh | 105 ++ web_interface/static/css/style.css | 817 +++++++++++++ web_interface/static/js/app.js | 1030 ++++++++++++++++ web_interface/templates/auth_error.html | 86 ++ web_interface/templates/auth_success.html | 82 ++ web_interface/templates/index.html | 209 ++++ 26 files changed, 6885 insertions(+), 3 deletions(-) create mode 100644 bdfr/API_README.md create mode 100644 bdfr/api.py create mode 100644 bdfr/examples/api_usage.py create mode 100644 tests/test_file_locking_fix.py create mode 100644 tests/test_user_folder_structure.py create mode 100644 web_interface/.dockerignore create mode 100644 web_interface/.env.example create mode 100644 web_interface/Dockerfile create mode 100644 web_interface/README.md create mode 100644 web_interface/STARTUP.md create mode 100644 web_interface/app/auth.py create mode 100644 web_interface/app/main.py create mode 100644 web_interface/requirements.txt create mode 100644 web_interface/setup_oauth.py create mode 100644 web_interface/start.bat create mode 100644 web_interface/start.py create mode 100644 web_interface/start.sh create mode 100644 web_interface/static/css/style.css create mode 100644 web_interface/static/js/app.js create mode 100644 web_interface/templates/auth_error.html create mode 100644 web_interface/templates/auth_success.html create mode 100644 web_interface/templates/index.html diff --git a/bdfr/API_README.md b/bdfr/API_README.md new file mode 100644 index 0000000..af23150 --- /dev/null +++ b/bdfr/API_README.md @@ -0,0 +1,430 @@ +# BDFR API Layer + +The BDFR API layer provides a direct integration interface for the web interface, eliminating the need for subprocess console parsing. It wraps the existing BDFR core classes and provides a clean API with structured progress callbacks. + +## Overview + +The API layer consists of several key components: + +- **BDFRManager**: Main API interface class +- **ProgressCallback**: Abstract base class for progress notifications +- **ProgressEvent**: Structured progress event data +- **DownloadType**: Enumeration of supported download types +- **DownloadStatus**: Enumeration of download statuses + +## Quick Start + +### Basic Usage + +```python +from bdfr.api import BDFRManager, LoggingCallback + +# Create a manager +manager = BDFRManager("./downloads") + +# Download from a subreddit +download_id = manager.download_subreddit( + "python", + limit=50, + sort="hot", + no_dupes=True +) + +# Check status +status = manager.get_download_status(download_id) +print(f"Progress: {status['progress']}%") +``` + +### Advanced Usage with Custom Callbacks + +```python +from bdfr.api import BDFRManager, ProgressCallback, ProgressEvent + +class WebSocketCallback(ProgressCallback): + def __init__(self, websocket): + self.websocket = websocket + + async def on_progress(self, event: ProgressEvent): + await self.websocket.send_json(event.to_dict()) + + async def on_error(self, event: ProgressEvent): + await self.websocket.send_json(event.to_dict()) + + async def on_completed(self, event: ProgressEvent): + await self.websocket.send_json(event.to_dict()) + +# Use custom callbacks +callbacks = [LoggingCallback(), WebSocketCallback(ws)] +download_id = manager.download_subreddit("technology", limit=100, progress_callbacks=callbacks) +``` + +## API Reference + +### BDFRManager + +The main API class that manages downloads and provides the primary interface. + +#### Constructor + +```python +BDFRManager(download_directory: Optional[Union[str, Path]] = None) +``` + +- `download_directory`: Base directory for downloads. Defaults to current directory. + +#### Methods + +##### `create_download()` + +Create a new download operation. + +```python +create_download( + download_type: DownloadType, + name: str, + config: Optional[Configuration] = None, + progress_callbacks: Optional[List[ProgressCallback]] = None +) -> str +``` + +Returns a download ID for tracking the operation. + +##### `start_download()` + +Start a download operation. + +```python +start_download(download_id: str) -> bool +``` + +Returns `True` if started successfully. + +##### `get_download_status()` + +Get the current status of a download. + +```python +get_download_status(download_id: str) -> Optional[Dict[str, Any]] +``` + +Returns download status information or `None` if not found. + +##### `cancel_download()` + +Cancel a running download. + +```python +cancel_download(download_id: str) -> bool +``` + +Returns `True` if cancelled successfully. + +##### `list_downloads()` + +List all active downloads. + +```python +list_downloads() -> Dict[str, Dict[str, Any]] +``` + +Returns dictionary of download information keyed by download ID. + +##### `cleanup_completed()` + +Clean up old completed downloads. + +```python +cleanup_completed(max_age_seconds: int = 3600) -> int +``` + +Returns number of downloads cleaned up. + +#### Convenience Methods + +##### `download_subreddit()` + +Download content from a subreddit. + +```python +download_subreddit( + subreddit_name: str, + limit: Optional[int] = None, + sort: str = "hot", + time_filter: str = "all", + no_dupes: bool = False, + progress_callbacks: Optional[List[ProgressCallback]] = None +) -> str +``` + +##### `download_user()` + +Download content from a user. + +```python +download_user( + username: str, + limit: Optional[int] = None, + submitted: bool = True, + upvoted: bool = False, + saved: bool = False, + no_dupes: bool = False, + progress_callbacks: Optional[List[ProgressCallback]] = None +) -> str +``` + +##### `archive_subreddit()` + +Archive subreddit data (metadata only). + +```python +archive_subreddit( + subreddit_name: str, + format_type: str = "json", + limit: Optional[int] = None, + progress_callbacks: Optional[List[ProgressCallback]] = None +) -> str +``` + +##### `clone_subreddit()` + +Clone subreddit (both download and archive). + +```python +clone_subreddit( + subreddit_name: str, + limit: Optional[int] = None, + format_type: str = "json", + no_dupes: bool = False, + progress_callbacks: Optional[List[ProgressCallback]] = None +) -> str +``` + +### ProgressCallback + +Abstract base class for implementing progress callbacks. + +#### Methods + +##### `on_progress(event: ProgressEvent)` + +Called when progress is made. + +##### `on_error(event: ProgressEvent)` + +Called when an error occurs. + +##### `on_completed(event: ProgressEvent)` + +Called when download is completed. + +### ProgressEvent + +Represents a progress event with structured data. + +#### Attributes + +- `event_type`: "progress", "status", "error", or "completed" +- `download_id`: Unique identifier for the download +- `message`: Human-readable message +- `progress`: Progress percentage (0-100) +- `data`: Additional structured data +- `timestamp`: When the event occurred + +#### Methods + +##### `to_dict() -> Dict[str, Any]` + +Convert to dictionary for JSON serialization. + +### DownloadType + +Enumeration of supported download types: + +- `SUBREDDIT`: Download from subreddit +- `USER`: Download from user +- `MULTIREDDIT`: Download from multireddit +- `SUBMISSIONS`: Download specific submissions +- `ARCHIVE`: Archive only (no downloads) +- `CLONE`: Both download and archive + +### DownloadStatus + +Enumeration of download statuses: + +- `QUEUED`: Download is queued but not started +- `RUNNING`: Download is in progress +- `COMPLETED`: Download completed successfully +- `FAILED`: Download failed with an error +- `CANCELLED`: Download was cancelled +- `PAUSED`: Download is paused + +## Configuration + +The API uses the existing BDFR `Configuration` class. You can pass a custom configuration to `create_download()` or use the convenience methods with their specific parameters. + +### Common Configuration Options + +- `limit`: Maximum number of posts to process +- `sort`: Sort method (hot, top, new, controversial, rising) +- `time`: Time filter (all, hour, day, week, month, year) +- `no_dupes`: Avoid duplicate downloads +- `make_hard_links`: Create hard links for duplicates +- `format`: Archive format (json, xml, yaml) + +## Web Interface Integration + +### FastAPI Integration Example + +```python +from fastapi import FastAPI, WebSocket +from bdfr.api import get_bdfr_manager, WebSocketCallback + +app = FastAPI() +manager = get_bdfr_manager("./downloads") + +@app.post("/api/download/subreddit") +async def download_subreddit(subreddit: str, limit: int = 10): + download_id = manager.download_subreddit(subreddit, limit=limit) + return {"download_id": download_id} + +@app.websocket("/ws/progress/{download_id}") +async def progress_websocket(websocket: WebSocket, download_id: str): + await websocket.accept() + + class FastAPICallback(WebSocketCallback): + def __init__(self): + super().__init__(None) + + async def on_progress(self, event: ProgressEvent): + if event.download_id == download_id: + await websocket.send_json(event.to_dict()) + + # Add callback to existing download or create new one + # (Implementation depends on your specific needs) +``` + +### Real-time Progress Updates + +The API provides structured progress events that can be easily serialized to JSON for web clients: + +```python +# Example progress event +{ + "event_type": "progress", + "download_id": "123e4567-e89b-12d3-a456-426614174000", + "message": "Downloaded submission abc123 from r/python", + "progress": 45.2, + "data": { + "items_processed": 12, + "items_found": 25, + "current_item": "abc123", + "phase": "downloading_submission" + }, + "timestamp": "2023-12-07T10:30:45.123456" +} +``` + +## Error Handling + +The API provides comprehensive error handling: + +- **Download errors**: Network issues, authentication problems, etc. +- **Configuration errors**: Invalid parameters, missing files, etc. +- **System errors**: Disk space, permissions, etc. + +All errors are captured and reported through the progress callback system with detailed error information. + +## Threading and Concurrency + +The API is designed to be thread-safe and supports concurrent downloads: + +- Each download runs in its own thread +- Progress callbacks are async-safe +- Multiple downloads can run simultaneously +- Thread-safe status tracking + +## Logging + +The API integrates with Python's logging system: + +```python +import logging + +# Configure logging +logging.basicConfig(level=logging.INFO) + +# Use LoggingCallback for automatic log output +callbacks = [LoggingCallback("my_app")] +``` + +## Examples + +See `bdfr/examples/api_usage.py` for comprehensive examples including: + +- Basic usage +- Custom callbacks +- Web integration +- Error handling +- User downloads +- Archive operations + +## Migration from Subprocess + +### Before (Subprocess) + +```python +import subprocess +import json + +# Start BDFR via subprocess +proc = subprocess.Popen([ + "python", "-m", "bdfr", "download", + "--subreddit", "python", + "--limit", "50", + "./downloads" +], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + +# Parse console output for progress +while True: + line = proc.stdout.readline().decode().strip() + if not line: + break + # Parse progress from console output... +``` + +### After (API) + +```python +from bdfr.api import BDFRManager, LoggingCallback + +# Use API directly +manager = BDFRManager("./downloads") +download_id = manager.download_subreddit("python", limit=50) + +# Get structured progress updates +status = manager.get_download_status(download_id) +print(f"Progress: {status['progress']}%") +``` + +## Benefits + +1. **No subprocess overhead**: Direct integration with BDFR core +2. **Structured progress**: Rich progress events instead of console parsing +3. **Better error handling**: Detailed error information and stack traces +4. **Thread-safe**: Concurrent downloads with proper synchronization +5. **Web-friendly**: JSON-serializable progress events +6. **Extensible**: Custom progress callbacks for different integrations +7. **Maintainable**: Clean separation of concerns + +## Requirements + +- Python 3.7+ +- Existing BDFR installation +- Dependencies: `praw`, `requests`, and other BDFR dependencies + +## Installation + +The API layer is included with BDFR and requires no additional installation. Simply import and use: + +```python +from bdfr.api import BDFRManager \ No newline at end of file diff --git a/bdfr/api.py b/bdfr/api.py new file mode 100644 index 0000000..bb7dd4b --- /dev/null +++ b/bdfr/api.py @@ -0,0 +1,1290 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +BDFR API Layer + +This module provides a direct integration interface for the web interface, +eliminating the need for subprocess console parsing. It wraps the existing +BDFR core classes and provides a clean API with structured progress callbacks. +""" + +import asyncio +import logging +import os +import tempfile +import threading +import time +import uuid +from abc import ABC, abstractmethod +from collections.abc import Callable, Iterable +from datetime import datetime +from enum import Enum +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +from bdfr.configuration import Configuration +from bdfr.connector import RedditConnector +from bdfr.downloader import RedditDownloader +from bdfr.archiver import Archiver +from bdfr.cloner import RedditCloner +from bdfr import exceptions as errors + +logger = logging.getLogger(__name__) + + +class DownloadType(Enum): + """Types of downloads supported by BDFR""" + SUBREDDIT = "subreddit" + USER = "user" + MULTIREDDIT = "multireddit" + SUBMISSIONS = "submissions" + ARCHIVE = "archive" + CLONE = "clone" + + +class DownloadStatus(Enum): + """Status of a download operation""" + QUEUED = "queued" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + PAUSED = "paused" + + +class ProgressEvent: + """Represents a progress event with structured data""" + + def __init__( + self, + event_type: str, + download_id: str, + message: str, + progress: Optional[float] = None, + data: Optional[Dict[str, Any]] = None, + timestamp: Optional[datetime] = None + ): + self.event_type = event_type # "progress", "status", "error", "completed" + self.download_id = download_id + self.message = message + self.progress = progress # 0-100 percentage + self.data = data or {} + self.timestamp = timestamp or datetime.now() + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for JSON serialization""" + return { + "event_type": self.event_type, + "download_id": self.download_id, + "message": self.message, + "progress": self.progress, + "data": self.data, + "timestamp": self.timestamp.isoformat() + } + + +class ProgressCallback(ABC): + """Abstract base class for progress callbacks""" + + @abstractmethod + async def on_progress(self, event: ProgressEvent): + """Called when progress is made""" + pass + + @abstractmethod + async def on_error(self, event: ProgressEvent): + """Called when an error occurs""" + pass + + @abstractmethod + async def on_completed(self, event: ProgressEvent): + """Called when download is completed""" + pass + + +class LoggingCallback(ProgressCallback): + """Default logging-based progress callback""" + + def __init__(self, logger_name: str = __name__): + self.logger = logging.getLogger(logger_name) + + async def on_progress(self, event: ProgressEvent): + """Log progress events""" + if event.progress is not None: + self.logger.info(f"[{event.download_id}] {event.message} ({event.progress}%)") + else: + self.logger.info(f"[{event.download_id}] {event.message}") + + async def on_error(self, event: ProgressEvent): + """Log error events""" + self.logger.error(f"[{event.download_id}] ERROR: {event.message}") + if event.data.get('exception'): + self.logger.exception(f"[{event.download_id}] Exception details", exc_info=event.data['exception']) + + async def on_completed(self, event: ProgressEvent): + """Log completion events""" + if event.event_type == "completed": + self.logger.info(f"[{event.download_id}] {event.message}") + else: + self.logger.info(f"[{event.download_id}] {event.message}") + + +class BDFRManager: + """ + Main API interface for BDFR operations. + + This class provides a clean interface for the web application to interact + with BDFR functionality without needing subprocess calls. + """ + + def __init__(self, download_directory: Optional[Union[str, Path]] = None, auth_token: Optional[str] = None): + """ + Initialize the BDFR Manager. + + Args: + download_directory: Base directory for downloads. If None, uses current directory. + auth_token: Optional authentication token for Reddit API + """ + self.download_directory = Path(download_directory or ".").resolve() + self.download_directory.mkdir(exist_ok=True, parents=True) + self.auth_token = auth_token + + # Active downloads tracking + self._active_downloads: Dict[str, Dict[str, Any]] = {} + self._download_threads: Dict[str, threading.Thread] = {} + self._callbacks: Dict[str, List[ProgressCallback]] = {} + + # Setup logging first + self.logger = logging.getLogger(__name__) + + # Default configuration + self._default_config = self._create_default_config() + + # Override the log file path to avoid conflicts between multiple BDFR instances + if not self._default_config.log: + # Create a unique log file for this BDFR manager instance + log_dir = self.download_directory / "logs" + log_dir.mkdir(exist_ok=True) + unique_log_file = log_dir / f"bdfr_manager_{int(time.time())}_{id(self)}.log" + self._default_config.log = str(unique_log_file) + + def _create_default_config(self) -> Configuration: + """Create a default configuration for BDFR operations""" + config = Configuration() + config.directory = str(self.download_directory) + config.limit = 100 # Default limit + config.sort = "hot" + config.time = "all" + config.no_dupes = False + config.make_hard_links = False + config.verbose = 1 + + # Set authentication if token provided + if self.auth_token: + config.authenticate = True + self.logger.info(f"[DEBUG] Authentication enabled for download with token") + else: + config.authenticate = False + self.logger.info(f"[DEBUG] No authentication token provided, using anonymous access") + + # Archiver defaults + config.format = "json" + config.all_comments = False + config.comment_context = False + + return config + + def create_download( + self, + download_type: DownloadType, + name: str, + config: Optional[Configuration] = None, + progress_callbacks: Optional[List[ProgressCallback]] = None + ) -> str: + """ + Create a new download operation. + + Args: + download_type: Type of download (subreddit, user, etc.) + name: Name of the target (subreddit name, username, etc.) + config: BDFR configuration. If None, uses defaults. + progress_callbacks: List of progress callbacks. If None, uses logging callback. + + Returns: + Download ID for tracking the operation + """ + download_id = str(uuid.uuid4()) + + # Use provided config or default + if config is None: + # Create a new Configuration instance with default values + config = Configuration() + config.directory = str(self.download_directory) + config.limit = 100 # Default limit + config.sort = "hot" + config.time = "all" + config.no_dupes = False + config.make_hard_links = False + config.verbose = 1 + config.format = "json" + config.all_comments = False + config.comment_context = False + + # Create unique log file for this download + log_dir = self.download_directory / "logs" + log_dir.mkdir(exist_ok=True) + unique_log_file = log_dir / f"bdfr_{download_type.value}_{name}_{download_id}_{int(time.time())}.log" + config.log = str(unique_log_file) + + # Set specific parameters based on download type + if download_type == DownloadType.SUBREDDIT: + config.subreddit = [name] + elif download_type == DownloadType.USER: + config.user = [name] + # Append username to directory path for user downloads + # This creates folder structure: downloads/username/subreddit/files + user_directory = self.download_directory / name + config.directory = str(user_directory) + elif download_type == DownloadType.MULTIREDDIT: + config.multireddit = [name] + + # Setup callbacks + if progress_callbacks is None: + progress_callbacks = [LoggingCallback()] + + self._callbacks[download_id] = progress_callbacks + + # Initialize download tracking + self._active_downloads[download_id] = { + "id": download_id, + "type": download_type.value, + "name": name, + "status": DownloadStatus.QUEUED.value, + "progress": 0.0, + "start_time": datetime.now(), + "config": config, + "callbacks": progress_callbacks, + "items_processed": 0, + "items_found": 0, + "current_item": None, + "phase": "queued" + } + + self.logger.info(f"Created download {download_id} for {download_type.value}: {name}") + return download_id + + def start_download(self, download_id: str) -> bool: + """ + Start a download operation asynchronously. + + Args: + download_id: ID of the download to start + + Returns: + True if started successfully, False if download not found or already running + """ + if download_id not in self._active_downloads: + self.logger.error(f"Download {download_id} not found") + return False + + download_info = self._active_downloads[download_id] + if download_info["status"] == DownloadStatus.RUNNING.value: + self.logger.warning(f"Download {download_id} is already running") + return False + + # Update status to running + download_info["status"] = DownloadStatus.RUNNING.value + download_info["start_time"] = datetime.now() + + # Create and start download thread + download_thread = threading.Thread( + target=self._run_download, + args=(download_id,), + daemon=True + ) + self._download_threads[download_id] = download_thread + download_thread.start() + + self.logger.info(f"Started download {download_id}") + return True + + def _run_download(self, download_id: str): + """Internal method to run a download in a separate thread""" + download_info = self._active_downloads[download_id] + config = download_info["config"] + callbacks = download_info["callbacks"] + downloader = None + + try: + self.logger.info(f"[DEBUG] Starting download {download_id} for {download_info['type']}: {download_info['name']}") + self.logger.info(f"[DEBUG] Config: subreddit={getattr(config, 'subreddit', None)}, user={getattr(config, 'user', None)}, limit={getattr(config, 'limit', None)}") + + # Create progress event for start + start_event = ProgressEvent( + "status", + download_id, + f"Starting {download_info['type']} download: {download_info['name']}", + 0.0, + {"phase": "starting"} + ) + + # Notify callbacks + self.logger.info(f"[BDFR-API] Notifying {len(callbacks)} callbacks of progress event for download {download_id}") + asyncio.run(self._notify_callbacks(callbacks, "on_progress", start_event)) + + # Set up authentication token in config if provided + if self.auth_token: + # Store the token in the config for the connector to use + config.auth_token = self.auth_token + self.logger.info(f"[DEBUG] Set auth token in config") + + # Ensure unique log file for this download to avoid conflicts + if not config.log: + # Create a unique log file for this download attempt + log_dir = self.download_directory / "logs" + log_dir.mkdir(exist_ok=True) + unique_log_file = log_dir / f"bdfr_download_{download_id}_{int(time.time())}.log" + config.log = str(unique_log_file) + + # Ensure log directory exists before creating downloader + if config.log: + log_path = Path(config.log) + log_path.parent.mkdir(parents=True, exist_ok=True) + + # Determine which BDFR class to use based on configuration + self.logger.info(f"[DEBUG] Determining downloader type - submitted={getattr(config, 'submitted', False)}, upvoted={getattr(config, 'upvoted', False)}, saved={getattr(config, 'saved', False)}, format={getattr(config, 'format', 'N/A')}") + + try: + if config.submitted or config.upvoted or config.saved: + # User data download + self.logger.info(f"[DEBUG] Creating RedditDownloader for user data") + downloader = RedditDownloader(config, []) + download_info["operation_type"] = "user_download" + self.logger.info(f"[DEBUG] Using RedditDownloader for user data") + elif hasattr(config, 'format') and config.format in ["json", "xml", "yaml"]: + # Archive operation + self.logger.info(f"[DEBUG] Creating Archiver for archive operation") + downloader = Archiver(config, []) + download_info["operation_type"] = "archive" + self.logger.info(f"[DEBUG] Using Archiver for archive operation") + else: + # Standard download + self.logger.info(f"[DEBUG] Creating RedditDownloader for standard download") + downloader = RedditDownloader(config, []) + download_info["operation_type"] = "download" + self.logger.info(f"[DEBUG] Using RedditDownloader for standard download") + except Exception as e: + self.logger.error(f"[DEBUG] Failed to create downloader: {e}") + raise + + # Run the actual download with progress tracking + self.logger.info(f"[DEBUG] Starting download_with_progress for {download_id}") + try: + self._download_with_progress(download_id, downloader, callbacks, config) + except Exception as e: + self.logger.error(f"[DEBUG] Exception in download_with_progress: {e}") + raise + + # Mark as completed only if not already failed or cancelled + if download_info.get("status") not in [DownloadStatus.FAILED.value, DownloadStatus.CANCELLED.value]: + self.logger.info(f"[DEBUG] Marking download {download_id} as completed") + self.logger.info(f"[DEBUG] Final stats - items_processed: {download_info['items_processed']}, items_found: {download_info['items_found']}") + download_info["status"] = DownloadStatus.COMPLETED.value + download_info["progress"] = 100.0 + download_info["end_time"] = datetime.now() + + completed_event = ProgressEvent( + "completed", + download_id, + f"Download completed successfully", + 100.0, + { + "items_processed": download_info["items_processed"], + "items_found": download_info["items_found"], + "operation_type": download_info.get("operation_type") + } + ) + + self.logger.info(f"[DEBUG] Sending completion callback for {download_id}") + self.logger.info(f"[BDFR-API] Sending completion event for download {download_id}") + asyncio.run(self._notify_callbacks(callbacks, "on_completed", completed_event)) + else: + self.logger.info(f"[DEBUG] Skipping completion for {download_id} due to status={download_info.get('status')}") + + except Exception as e: + import traceback + # Get full stack trace for debugging + stack_trace = traceback.format_exc() + + # Determine if this failure was due to rate limiting (HTTP 429) + is_rate_limited = False + try: + import prawcore + TooManyRequests = getattr(prawcore.exceptions, "TooManyRequests", None) + if TooManyRequests is not None and isinstance(e, TooManyRequests): + is_rate_limited = True + except Exception: + pass + if "429" in str(e): + is_rate_limited = True + + # Mark as failed + download_info["status"] = DownloadStatus.FAILED.value + download_info["error"] = str(e) + download_info["stack_trace"] = stack_trace + download_info["end_time"] = datetime.now() + + error_event = ProgressEvent( + "error", + download_id, + f"Download failed: {str(e)}", + None, + {"exception": str(e), "stack_trace": stack_trace, "phase": ("rate_limited" if is_rate_limited else "failed")} + ) + + asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event)) + + finally: + # Clean up thread reference + if download_id in self._download_threads: + del self._download_threads[download_id] + + # Clean up logging handlers to prevent file locking issues + if downloader is not None: + try: + # Close any logging handlers that might be holding file locks + logger = logging.getLogger() + for handler in logger.handlers[:]: + if hasattr(handler, 'close') and hasattr(handler, 'baseFilename'): + # Check if this handler is for our download's log file + if hasattr(config, 'log') and config.log: + handler_path = getattr(handler, 'baseFilename', '') + if handler_path == config.log or handler_path.startswith(str(self.download_directory)): + try: + handler.close() + logger.removeHandler(handler) + self.logger.info(f"[DEBUG] Closed logging handler for {handler_path}") + except Exception as cleanup_error: + self.logger.warning(f"[DEBUG] Error closing handler {handler_path}: {cleanup_error}") + except Exception as cleanup_error: + self.logger.warning(f"[DEBUG] Error during logging cleanup: {cleanup_error}") + + def _download_with_progress(self, download_id: str, downloader, callbacks: List[ProgressCallback], config): + """Run download with progress tracking and callbacks""" + download_info = self._active_downloads[download_id] + + # Create a progress-tracking wrapper for the downloader + original_download = downloader.download + + def progress_download(): + """Wrapper for download method that sends progress updates""" + try: + # Initialize submission variable before any potential exceptions + submission = None + + # Update phase to fetching + download_info["phase"] = "fetching_submissions" + progress_event = ProgressEvent( + "progress", + download_id, + "Fetching submissions...", + 10.0, + {"phase": "fetching_submissions"} + ) + self.logger.info(f"[BDFR-API] Sending progress update for download {download_id}") + asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) + + # For RedditDownloader, we need to intercept the download process + if hasattr(downloader, 'reddit_lists') and hasattr(downloader, '_download_submission'): + processed_submissions = 0 + + self.logger.info(f"[DEBUG] RedditDownloader has reddit_lists and _download_submission methods") + + # Get the limit from config - this is our target/goal for items_found + # items_found represents "how many items are available to download (up to limit)" + # items_processed represents "how many we've actually processed so far" + limit = getattr(config, 'limit', 100) + download_info["items_found"] = limit + self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}") + + # Process submissions with progress tracking + self.logger.info(f"[DEBUG] Starting submission processing loop...") + actual_count = 0 # Track actual submissions found for final adjustment + + for generator in downloader.reddit_lists: + self.logger.info(f"[DEBUG] Processing generator...") + submission_count = 0 + for submission in generator: + # Ensure submission is defined for the entire loop + if submission is None: + continue + submission_count += 1 + actual_count += 1 + + if download_info["status"] == DownloadStatus.CANCELLED.value: + self.logger.info(f"Download {download_id} was cancelled") + return + + try: + self.logger.info(f"[DEBUG] Processing submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}") + + # Update current item being processed + download_info["current_item"] = submission.id + download_info["current_item_type"] = "submission" + download_info["current_subreddit"] = submission.subreddit.display_name + + # Send progress update for this submission + processed_submissions += 1 + # Calculate progress based on processed vs expected (limit), but cap at 90% + # This gives smooth progress even when we don't know the exact total yet + progress_percent = 20.0 + (processed_submissions / max(limit, 1)) * 70.0 # 20-90% range + + progress_event = ProgressEvent( + "progress", + download_id, + f"Processing submission: {submission.id}", + min(progress_percent, 90.0), + { + "phase": "downloading_submission", + "current_item": submission.id, + "current_item_type": "submission", + "current_subreddit": submission.subreddit.display_name, + "items_processed": processed_submissions, + "items_found": download_info["items_found"] # Keep showing the limit as target + } + ) + asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) + + # Process this submission + self.logger.info(f"[DEBUG] Calling _download_submission for {submission.id}") + downloader._download_submission(submission) + self.logger.info(f"[DEBUG] Completed _download_submission for {submission.id}") + + # Update processed count + download_info["items_processed"] = processed_submissions + + except Exception as e: + import traceback + error_msg = str(e) + submission_id = submission.id if submission is not None else "unknown" + + # Get full stack trace for debugging + stack_trace = traceback.format_exc() + + if "429" in error_msg: + self.logger.error(f"[DEBUG] Rate limited while processing submission {submission_id}: {e}") + self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") + # Mark download as failed due to rate limiting + download_info["status"] = DownloadStatus.FAILED.value + download_info["error"] = f"Rate limited by Reddit API: {e}" + download_info["end_time"] = datetime.now() + + error_event = ProgressEvent( + "error", + download_id, + f"Rate limited by Reddit API: {e}", + None, + {"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"} + ) + asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event)) + return + else: + self.logger.error(f"[DEBUG] Error processing submission {submission_id}: {e}") + self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") + continue + + self.logger.info(f"[DEBUG] Generator processed {submission_count} submissions") + + # Final progress update + # If we found fewer items than the limit, update items_found to reflect reality + if actual_count < limit: + download_info["items_found"] = actual_count + self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}") + + self.logger.info(f"[DEBUG] Final progress update - processed {processed_submissions} out of {download_info['items_found']} found") + + progress_event = ProgressEvent( + "progress", + download_id, + "Download completed", + 100.0, + { + "phase": "completed", + "items_processed": processed_submissions, + "items_found": download_info["items_found"] + } + ) + asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) + + elif hasattr(downloader, 'reddit_lists') and hasattr(downloader, 'write_entry'): + # For Archiver, we need to intercept the archiving process + processed_items = 0 + + self.logger.info(f"[DEBUG] Archiver detected with reddit_lists and write_entry methods") + + # Get the limit from config + limit = getattr(config, 'limit', 100) + download_info["items_found"] = limit + self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}") + + # Process items with progress tracking + self.logger.info(f"[DEBUG] Starting archiving loop...") + actual_count = 0 + + for generator in downloader.reddit_lists: + self.logger.info(f"[DEBUG] Processing generator for archiving...") + item_count = 0 + for item in generator: + if item is None: + continue + item_count += 1 + actual_count += 1 + + if download_info["status"] == DownloadStatus.CANCELLED.value: + self.logger.info(f"Download {download_id} was cancelled") + return + + try: + # Determine item type + import praw.models + if isinstance(item, praw.models.Submission): + item_id = item.id + item_type = "submission" + subreddit_name = item.subreddit.display_name + elif isinstance(item, praw.models.Comment): + item_id = item.id + item_type = "comment" + subreddit_name = item.subreddit.display_name if hasattr(item, 'subreddit') else "unknown" + else: + item_id = str(item) + item_type = "item" + subreddit_name = "unknown" + + self.logger.info(f"[DEBUG] Archiving {item_type} {item_count}: {item_id} from r/{subreddit_name}") + + # Update current item being processed + download_info["current_item"] = item_id + download_info["current_item_type"] = item_type + download_info["current_subreddit"] = subreddit_name + + # Send progress update for this item + processed_items += 1 + progress_percent = 20.0 + (processed_items / max(limit, 1)) * 70.0 + + progress_event = ProgressEvent( + "progress", + download_id, + f"Archiving {item_type}: {item_id}", + min(progress_percent, 90.0), + { + "phase": "archiving_item", + "current_item": item_id, + "current_item_type": item_type, + "current_subreddit": subreddit_name, + "items_processed": processed_items, + "items_found": download_info["items_found"] + } + ) + asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) + + # Archive this item + self.logger.info(f"[DEBUG] Calling write_entry for {item_id}") + downloader.write_entry(item) + self.logger.info(f"[DEBUG] Completed write_entry for {item_id}") + + # Update processed count + download_info["items_processed"] = processed_items + + except Exception as e: + import traceback + error_msg = str(e) + item_id = item_id if 'item_id' in locals() else "unknown" + stack_trace = traceback.format_exc() + + if "429" in error_msg: + self.logger.error(f"[DEBUG] Rate limited while archiving item {item_id}: {e}") + self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") + download_info["status"] = DownloadStatus.FAILED.value + download_info["error"] = f"Rate limited by Reddit API: {e}" + download_info["end_time"] = datetime.now() + + error_event = ProgressEvent( + "error", + download_id, + f"Rate limited by Reddit API: {e}", + None, + {"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"} + ) + asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event)) + return + else: + self.logger.error(f"[DEBUG] Error archiving item {item_id}: {e}") + self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") + continue + + self.logger.info(f"[DEBUG] Generator archived {item_count} items") + + # Final progress update + if actual_count < limit: + download_info["items_found"] = actual_count + self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}") + + self.logger.info(f"[DEBUG] Final progress update - archived {processed_items} out of {download_info['items_found']} found") + + progress_event = ProgressEvent( + "progress", + download_id, + "Archiving completed", + 100.0, + { + "phase": "completed", + "items_processed": processed_items, + "items_found": download_info["items_found"] + } + ) + asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) + + elif hasattr(downloader, 'reddit_lists') and hasattr(downloader, '_download_submission') and hasattr(downloader, 'write_entry'): + # For RedditCloner (which has both download and archive capabilities) + processed_items = 0 + + self.logger.info(f"[DEBUG] RedditCloner detected with reddit_lists, _download_submission, and write_entry methods") + + # Get the limit from config + limit = getattr(config, 'limit', 100) + download_info["items_found"] = limit + self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}") + + # Process submissions with progress tracking (both download and archive) + self.logger.info(f"[DEBUG] Starting cloning loop...") + actual_count = 0 + + for generator in downloader.reddit_lists: + self.logger.info(f"[DEBUG] Processing generator for cloning...") + submission_count = 0 + for submission in generator: + if submission is None: + continue + submission_count += 1 + actual_count += 1 + + if download_info["status"] == DownloadStatus.CANCELLED.value: + self.logger.info(f"Download {download_id} was cancelled") + return + + try: + self.logger.info(f"[DEBUG] Cloning submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}") + + # Update current item being processed + download_info["current_item"] = submission.id + download_info["current_item_type"] = "submission" + download_info["current_subreddit"] = submission.subreddit.display_name + + # Send progress update for this submission + processed_items += 1 + progress_percent = 20.0 + (processed_items / max(limit, 1)) * 70.0 + + progress_event = ProgressEvent( + "progress", + download_id, + f"Cloning submission: {submission.id}", + min(progress_percent, 90.0), + { + "phase": "cloning_submission", + "current_item": submission.id, + "current_item_type": "submission", + "current_subreddit": submission.subreddit.display_name, + "items_processed": processed_items, + "items_found": download_info["items_found"] + } + ) + asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) + + # Clone this submission (download + archive) + self.logger.info(f"[DEBUG] Calling _download_submission for {submission.id}") + downloader._download_submission(submission) + self.logger.info(f"[DEBUG] Calling write_entry for {submission.id}") + downloader.write_entry(submission) + self.logger.info(f"[DEBUG] Completed cloning for {submission.id}") + + # Update processed count + download_info["items_processed"] = processed_items + + except Exception as e: + import traceback + error_msg = str(e) + submission_id = submission.id if submission is not None else "unknown" + stack_trace = traceback.format_exc() + + if "429" in error_msg: + self.logger.error(f"[DEBUG] Rate limited while cloning submission {submission_id}: {e}") + self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") + download_info["status"] = DownloadStatus.FAILED.value + download_info["error"] = f"Rate limited by Reddit API: {e}" + download_info["end_time"] = datetime.now() + + error_event = ProgressEvent( + "error", + download_id, + f"Rate limited by Reddit API: {e}", + None, + {"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"} + ) + asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event)) + return + else: + self.logger.error(f"[DEBUG] Error cloning submission {submission_id}: {e}") + self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") + continue + + self.logger.info(f"[DEBUG] Generator cloned {submission_count} submissions") + + # Final progress update + if actual_count < limit: + download_info["items_found"] = actual_count + self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}") + + self.logger.info(f"[DEBUG] Final progress update - cloned {processed_items} out of {download_info['items_found']} found") + + progress_event = ProgressEvent( + "progress", + download_id, + "Cloning completed", + 100.0, + { + "phase": "completed", + "items_processed": processed_items, + "items_found": download_info["items_found"] + } + ) + asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event)) + + else: + # For other downloaders, just run normally + self.logger.info(f"[DEBUG] Using fallback downloader (no specific handler)") + original_download() + + except Exception as e: + import traceback + stack_trace = traceback.format_exc() + self.logger.error(f"Error in progress download for {download_id}: {e}") + self.logger.error(f"[DEBUG] Stack trace: {stack_trace}") + raise + + # Set up authentication if token provided + if hasattr(config, 'auth_token') and config.auth_token: + self.logger.info(f"[DEBUG] Setting up authenticated Reddit instance") + # Monkey patch the create_reddit_instance method to use our token + original_create_reddit_instance = downloader.create_reddit_instance + auth_token = config.auth_token # Capture token in closure + + def authenticated_create_reddit_instance(): + try: + self.logger.info(f"[DEBUG] Creating authenticated Reddit instance") + # Call original method first to set up basic config + original_create_reddit_instance() + + # Override the reddit_instance with authenticated version + import praw + downloader.reddit_instance = praw.Reddit( + client_id=downloader.cfg_parser.get("DEFAULT", "client_id"), + client_secret=downloader.cfg_parser.get("DEFAULT", "client_secret"), + user_agent=getattr(downloader, 'user_agent', 'BDFR-Web-Interface/1.0'), + token=auth_token + ) + downloader.authenticated = True + self.logger.info(f"[DEBUG] Created authenticated Reddit instance successfully") + except Exception as e: + self.logger.error(f"[DEBUG] Failed to create authenticated Reddit instance: {e}") + raise + + downloader.create_reddit_instance = authenticated_create_reddit_instance + + # Replace the download method and run it + downloader.download = progress_download + downloader.download() + + async def _notify_callbacks(self, callbacks: List[ProgressCallback], method_name: str, event: ProgressEvent): + """Notify all callbacks of an event""" + self.logger.info(f"[BDFR-API] Notifying {len(callbacks)} callbacks of {method_name} event for download {event.download_id}") + for callback in callbacks: + try: + logger.info(f"[BDFR-API] Invoking {method_name} on callback {callback.__class__.__name__} for download {event.download_id}") + method = getattr(callback, method_name) + if asyncio.iscoroutinefunction(method): + await method(event) + else: + # Run sync method in thread pool + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, lambda: asyncio.run(method(event))) + logger.info(f"[BDFR-API] Successfully invoked {method_name} on callback {callback.__class__.__name__}") + except Exception as e: + self.logger.error(f"[BDFR-API] Error in callback {callback.__class__.__name__}: {e}") + + def get_download_status(self, download_id: str) -> Optional[Dict[str, Any]]: + """ + Get the current status of a download. + + Args: + download_id: ID of the download to check + + Returns: + Download status information or None if not found + """ + if download_id not in self._active_downloads: + return None + + download_info = dict(self._active_downloads[download_id]) + + # Add thread status if running + if download_id in self._download_threads: + thread = self._download_threads[download_id] + download_info["thread_alive"] = thread.is_alive() + + return download_info + + def cancel_download(self, download_id: str) -> bool: + """ + Cancel a running download. + + Args: + download_id: ID of the download to cancel + + Returns: + True if cancelled successfully, False if download not found or not running + """ + if download_id not in self._active_downloads: + return False + + download_info = self._active_downloads[download_id] + + if download_info["status"] not in [DownloadStatus.RUNNING.value, DownloadStatus.QUEUED.value]: + return False + + # Mark as cancelled + download_info["status"] = DownloadStatus.CANCELLED.value + download_info["end_time"] = datetime.now() + + # Try to stop the thread (this is a best effort) + if download_id in self._download_threads: + thread = self._download_threads[download_id] + # Note: We can't actually kill the thread safely, but we can mark it as cancelled + # The BDFR instance would need to check for cancellation periodically + + self.logger.info(f"Cancelled download {download_id}") + return True + + def list_downloads(self) -> Dict[str, Dict[str, Any]]: + """ + List all downloads (active and recently completed). + + Returns: + Dictionary of download information keyed by download ID + """ + return { + download_id: self.get_download_status(download_id) + for download_id in self._active_downloads + } + + def cleanup_completed(self, max_age_seconds: int = 3600) -> int: + """ + Clean up completed/failed downloads older than specified age. + + Args: + max_age_seconds: Maximum age in seconds for completed downloads to keep + + Returns: + Number of downloads cleaned up + """ + current_time = datetime.now() + to_remove = [] + + for download_id, download_info in self._active_downloads.items(): + if download_info["status"] in [DownloadStatus.COMPLETED.value, DownloadStatus.FAILED.value, DownloadStatus.CANCELLED.value]: + end_time = download_info.get("end_time") + if end_time and (current_time - end_time).total_seconds() > max_age_seconds: + to_remove.append(download_id) + + for download_id in to_remove: + del self._active_downloads[download_id] + if download_id in self._callbacks: + del self._callbacks[download_id] + + if to_remove: + self.logger.info(f"Cleaned up {len(to_remove)} old downloads: {to_remove}") + + return len(to_remove) + + # Convenience methods for common operations + + def download_subreddit( + self, + subreddit_name: str, + limit: Optional[int] = None, + sort: str = "hot", + time_filter: str = "all", + no_dupes: bool = False, + simple_check: bool = False, + progress_callbacks: Optional[List[ProgressCallback]] = None + ) -> str: + """ + Download content from a subreddit. + + Args: + subreddit_name: Name of the subreddit (without r/) + limit: Maximum number of posts to download + sort: Sort method (hot, top, new, controversial, rising) + time_filter: Time filter (all, hour, day, week, month, year) + no_dupes: Whether to avoid duplicate downloads + simple_check: Whether to use simple URL-based duplicate checking + progress_callbacks: Optional progress callbacks + + Returns: + Download ID for tracking + """ + # Create a new Configuration instance with default values + config = Configuration() + config.directory = str(self.download_directory) + config.limit = 100 # Default limit + config.sort = "hot" + config.time = "all" + config.no_dupes = False + config.make_hard_links = False + config.verbose = 1 + config.format = "json" + config.all_comments = False + config.comment_context = False + + # Create unique log file for this download + log_dir = self.download_directory / "logs" + log_dir.mkdir(exist_ok=True) + unique_log_file = log_dir / f"bdfr_subreddit_{subreddit_name}_{int(time.time())}.log" + config.log = str(unique_log_file) + + config.subreddit = [subreddit_name] + if limit: + config.limit = limit + config.sort = sort + config.time = time_filter + config.no_dupes = no_dupes + config.simple_check = simple_check + + download_id = self.create_download( + DownloadType.SUBREDDIT, + subreddit_name, + config, + progress_callbacks + ) + + self.start_download(download_id) + return download_id + + def download_user( + self, + username: str, + limit: Optional[int] = None, + submitted: bool = True, + upvoted: bool = False, + saved: bool = False, + no_dupes: bool = False, + simple_check: bool = False, + progress_callbacks: Optional[List[ProgressCallback]] = None + ) -> str: + """ + Download content from a user. + + Args: + username: Reddit username + limit: Maximum number of posts to download + submitted: Download user's submitted posts + upvoted: Download user's upvoted posts (requires authentication) + saved: Download user's saved posts (requires authentication) + no_dupes: Whether to avoid duplicate downloads + progress_callbacks: Optional progress callbacks + + Returns: + Download ID for tracking + """ + # Create a new Configuration instance with default values + config = Configuration() + # Append username to directory path for user downloads + # This creates folder structure: downloads/username/subreddit/files + user_directory = self.download_directory / username + config.directory = str(user_directory) + config.limit = 100 # Default limit + config.sort = "hot" + config.time = "all" + config.no_dupes = False + config.make_hard_links = False + config.verbose = 1 + config.format = "json" + config.all_comments = False + config.comment_context = False + + # Create unique log file for this download + log_dir = self.download_directory / "logs" + log_dir.mkdir(exist_ok=True) + unique_log_file = log_dir / f"bdfr_user_{username}_{int(time.time())}.log" + config.log = str(unique_log_file) + + config.user = [username] + if limit: + config.limit = limit + config.submitted = submitted + config.upvoted = upvoted + config.saved = saved + config.no_dupes = no_dupes + config.simple_check = simple_check + + download_id = self.create_download( + DownloadType.USER, + username, + config, + progress_callbacks + ) + + self.start_download(download_id) + return download_id + + def archive_subreddit( + self, + subreddit_name: str, + format_type: str = "json", + limit: Optional[int] = None, + simple_check: bool = False, + progress_callbacks: Optional[List[ProgressCallback]] = None + ) -> str: + """ + Archive subreddit data (metadata only, no downloads). + + Args: + subreddit_name: Name of the subreddit + format_type: Archive format (json, xml, yaml) + limit: Maximum number of posts to archive + progress_callbacks: Optional progress callbacks + + Returns: + Download ID for tracking + """ + # Create a new Configuration instance with default values + config = Configuration() + config.directory = str(self.download_directory) + config.limit = 100 # Default limit + config.sort = "hot" + config.time = "all" + config.no_dupes = False + config.make_hard_links = False + config.verbose = 1 + config.format = "json" + config.all_comments = False + config.comment_context = False + + # Create unique log file for this download + log_dir = self.download_directory / "logs" + log_dir.mkdir(exist_ok=True) + unique_log_file = log_dir / f"bdfr_archive_{subreddit_name}_{int(time.time())}.log" + config.log = str(unique_log_file) + + config.subreddit = [subreddit_name] + if limit: + config.limit = limit + config.format = format_type + + download_id = self.create_download( + DownloadType.ARCHIVE, + subreddit_name, + config, + progress_callbacks + ) + + self.start_download(download_id) + return download_id + + def clone_subreddit( + self, + subreddit_name: str, + limit: Optional[int] = None, + format_type: str = "json", + no_dupes: bool = False, + simple_check: bool = False, + progress_callbacks: Optional[List[ProgressCallback]] = None + ) -> str: + """ + Clone subreddit (both download and archive). + + Args: + subreddit_name: Name of the subreddit + limit: Maximum number of posts to process + format_type: Archive format for metadata + no_dupes: Whether to avoid duplicate downloads + progress_callbacks: Optional progress callbacks + + Returns: + Download ID for tracking + """ + # Create a new Configuration instance with default values + config = Configuration() + config.directory = str(self.download_directory) + config.limit = 100 # Default limit + config.sort = "hot" + config.time = "all" + config.no_dupes = False + config.make_hard_links = False + config.verbose = 1 + config.format = "json" + config.all_comments = False + config.comment_context = False + + # Create unique log file for this download + log_dir = self.download_directory / "logs" + log_dir.mkdir(exist_ok=True) + unique_log_file = log_dir / f"bdfr_clone_{subreddit_name}_{int(time.time())}.log" + config.log = str(unique_log_file) + + config.subreddit = [subreddit_name] + if limit: + config.limit = limit + config.format = format_type + config.no_dupes = no_dupes + + download_id = self.create_download( + DownloadType.CLONE, + subreddit_name, + config, + progress_callbacks + ) + + self.start_download(download_id) + return download_id + + +# Global instance for easy access +_default_manager: Optional[BDFRManager] = None + + +def get_bdfr_manager(download_directory: Optional[Union[str, Path]] = None) -> BDFRManager: + """ + Get the default BDFR manager instance. + + Args: + download_directory: Base directory for downloads + + Returns: + BDFRManager instance + """ + global _default_manager + if _default_manager is None: + _default_manager = BDFRManager(download_directory) + return _default_manager + + +def set_bdfr_manager(manager: BDFRManager): + """ + Set the default BDFR manager instance. + + Args: + manager: BDFRManager instance to use as default + """ + global _default_manager + _default_manager = manager \ No newline at end of file diff --git a/bdfr/archiver.py b/bdfr/archiver.py index 52b4649..dec1c64 100644 --- a/bdfr/archiver.py +++ b/bdfr/archiver.py @@ -31,6 +31,7 @@ class Archiver(RedditConnector): def download(self): for generator in self.reddit_lists: + submission = None try: for submission in generator: try: @@ -50,7 +51,10 @@ class Archiver(RedditConnector): except prawcore.PrawcoreException as e: logger.error(f"Submission {submission.id} failed to be archived due to a PRAW exception: {e}") except prawcore.PrawcoreException as e: - logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}") + if submission is not None: + logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}") + else: + logger.error(f"Download failed due to a PRAW exception: {e}") logger.debug("Waiting 60 seconds to continue") sleep(60) diff --git a/bdfr/cloner.py b/bdfr/cloner.py index df71c28..8376199 100644 --- a/bdfr/cloner.py +++ b/bdfr/cloner.py @@ -20,6 +20,7 @@ class RedditCloner(RedditDownloader, Archiver): def download(self): for generator in self.reddit_lists: + submission = None try: for submission in generator: try: @@ -28,6 +29,9 @@ class RedditCloner(RedditDownloader, Archiver): except prawcore.PrawcoreException as e: logger.error(f"Submission {submission.id} failed to be cloned due to a PRAW exception: {e}") except prawcore.PrawcoreException as e: - logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}") + if submission is not None: + logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}") + else: + logger.error(f"Download failed due to a PRAW exception: {e}") logger.debug("Waiting 60 seconds to continue") sleep(60) diff --git a/bdfr/connector.py b/bdfr/connector.py index 77a4a71..0a1605f 100644 --- a/bdfr/connector.py +++ b/bdfr/connector.py @@ -386,6 +386,18 @@ class RedditConnector(metaclass=ABCMeta): generators.append(self.reddit_instance.redditor(user).saved(limit=self.args.limit)) except prawcore.PrawcoreException as e: logger.error(f"User {user} failed to be retrieved due to a PRAW exception: {e}") + # Detect HTTP 429 (rate limiting) and propagate as a hard failure so the UI can show 'failed' + TooManyRequests = getattr(prawcore.exceptions, "TooManyRequests", None) + is_rate_limited = False + if TooManyRequests is not None and isinstance(e, TooManyRequests): + is_rate_limited = True + elif (hasattr(e, "response") and getattr(e.response, "status_code", None) == 429) or "429" in str(e): + is_rate_limited = True + + if is_rate_limited: + logger.error("Received HTTP 429 (rate limited). Propagating error to fail the download.") + raise + logger.debug("Waiting 60 seconds to continue") sleep(60) return generators diff --git a/bdfr/downloader.py b/bdfr/downloader.py index 22a5a11..fe12bc2 100644 --- a/bdfr/downloader.py +++ b/bdfr/downloader.py @@ -84,7 +84,7 @@ class RedditDownloader(RedditConnector): except prawcore.PrawcoreException as e: logger.error(f"Submission {submission.id} failed to download due to a PRAW exception: {e}") except prawcore.PrawcoreException as e: - submission_id = last_submission_id or "unknown" + submission_id = last_submission_id if last_submission_id is not None else "unknown" logger.error(f"The submission after {submission_id} failed to download due to a PRAW exception: {e}") logger.debug("Waiting 60 seconds to continue") sleep(60) diff --git a/bdfr/examples/api_usage.py b/bdfr/examples/api_usage.py new file mode 100644 index 0000000..9fb47b2 --- /dev/null +++ b/bdfr/examples/api_usage.py @@ -0,0 +1,409 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +BDFR API Usage Examples + +This file demonstrates how to use the BDFR API layer for direct integration +with the web interface, eliminating the need for subprocess console parsing. +""" + +import asyncio +import logging +import sys +from pathlib import Path +from typing import List + +# Add the parent directory to the path so we can import bdfr +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from bdfr.api import ( + BDFRManager, + DownloadType, + DownloadStatus, + ProgressEvent, + ProgressCallback, + LoggingCallback, + get_bdfr_manager +) + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='[%(asctime)s] %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +) +logger = logging.getLogger(__name__) + + +class WebSocketCallback(ProgressCallback): + """Example callback that simulates WebSocket updates""" + + def __init__(self, websocket_id: str = "demo"): + self.websocket_id = websocket_id + + async def on_progress(self, event: ProgressEvent): + """Send progress update to WebSocket""" + print(f"📊 [{self.websocket_id}] Progress: {event.message}") + if event.progress is not None: + print(f" Progress: {event.progress:.1f}%") + if event.data: + print(f" Data: {event.data}") + + async def on_error(self, event: ProgressEvent): + """Send error update to WebSocket""" + print(f"❌ [{self.websocket_id}] ERROR: {event.message}") + if event.data.get('exception'): + print(f" Exception: {event.data['exception']}") + + async def on_completed(self, event: ProgressEvent): + """Send completion update to WebSocket""" + status_icon = "✅" if event.event_type == "completed" else "⚠️" + print(f"{status_icon} [{self.websocket_id}] {event.message}") + if event.data: + print(f" Final stats: {event.data}") + + +class DatabaseCallback(ProgressCallback): + """Example callback that saves progress to a database""" + + def __init__(self, db_connection_string: str = "sqlite:///progress.db"): + self.db_connection = db_connection_string + + async def on_progress(self, event: ProgressEvent): + """Save progress to database""" + # In a real implementation, you would save to your database + print(f"💾 [DB] Saved progress for {event.download_id}: {event.progress}%") + + async def on_error(self, event: ProgressEvent): + """Save error to database""" + print(f"💾 [DB] Saved error for {event.download_id}: {event.message}") + + async def on_completed(self, event: ProgressEvent): + """Save completion to database""" + print(f"💾 [DB] Saved completion for {event.download_id}") + + +async def example_basic_usage(): + """Basic usage example""" + print("🚀 Basic BDFR API Usage Example") + print("=" * 50) + + # Create a BDFR manager + manager = BDFRManager("./downloads") + + # Create a download for a subreddit + download_id = manager.download_subreddit( + "python", # subreddit name + limit=10, # download 10 posts + sort="hot", # sort by hot + no_dupes=True # avoid duplicates + ) + + print(f"📋 Created download with ID: {download_id}") + + # Monitor progress + while True: + status = manager.get_download_status(download_id) + if not status: + print("❌ Download not found!") + break + + print(f"📊 Status: {status['status']} | Progress: {status['progress']:.1f}%") + + if status['status'] in ['completed', 'failed', 'cancelled']: + print(f"🏁 Download finished with status: {status['status']}") + break + + await asyncio.sleep(2) # Check every 2 seconds + + return download_id + + +async def example_advanced_usage(): + """Advanced usage with custom callbacks""" + print("\n🎯 Advanced BDFR API Usage Example") + print("=" * 50) + + # Create custom callbacks + callbacks = [ + LoggingCallback("web_interface"), + WebSocketCallback("user_123"), + DatabaseCallback() + ] + + # Create manager with custom download directory + manager = BDFRManager("./custom_downloads") + + # Download from multiple subreddits + subreddits = ["programming", "learnprogramming", "Python"] + download_ids = [] + + for subreddit in subreddits: + download_id = manager.create_download( + DownloadType.SUBREDDIT, + subreddit, + progress_callbacks=callbacks + ) + + # Start the download + manager.start_download(download_id) + download_ids.append(download_id) + print(f"📋 Started download {download_id} for r/{subreddit}") + + # Monitor all downloads + while download_ids: + active_downloads = [] + + for download_id in download_ids[:]: # Copy list to avoid modification during iteration + status = manager.get_download_status(download_id) + if not status: + print(f"❌ Download {download_id} not found") + download_ids.remove(download_id) + continue + + print(f"📊 {download_id}: {status['status']} ({status['progress']:.1f}%)") + + if status['status'] in ['completed', 'failed', 'cancelled']: + print(f"🏁 Download {download_id} finished") + download_ids.remove(download_id) + else: + active_downloads.append(download_id) + + if not active_downloads: + break + + await asyncio.sleep(3) # Check every 3 seconds + + return len(download_ids) == 0 # Return success status + + +async def example_user_download(): + """Example of downloading user content""" + print("\n👤 User Download Example") + print("=" * 50) + + manager = get_bdfr_manager() # Use default manager + + # Download user's submitted posts + download_id = manager.download_user( + "testuser", # username + limit=25, # 25 posts + submitted=True, + upvoted=False, + saved=False + ) + + print(f"📋 Created user download: {download_id}") + + # Check status periodically + for _ in range(10): # Check for up to 20 seconds + status = manager.get_download_status(download_id) + if not status: + print("❌ Download not found") + break + + print(f"📊 Status: {status['status']} | Progress: {status['progress']:.1f}%") + + if status['status'] in ['completed', 'failed']: + break + + await asyncio.sleep(2) + + return download_id + + +async def example_archive_operation(): + """Example of archiving subreddit data""" + print("\n📚 Archive Operation Example") + print("=" * 50) + + manager = BDFRManager("./archives") + + # Archive subreddit data (metadata only) + download_id = manager.archive_subreddit( + "dataisbeautiful", + format_type="json", + limit=50 + ) + + print(f"📋 Created archive operation: {download_id}") + + # Monitor progress + while True: + status = manager.get_download_status(download_id) + if not status: + print("❌ Archive not found") + break + + print(f"📊 Archive status: {status['status']} | Progress: {status['progress']:.1f}%") + + if status['status'] in ['completed', 'failed']: + print(f"🏁 Archive finished with status: {status['status']}") + break + + await asyncio.sleep(2) + + return download_id + + +async def example_web_integration(): + """Example showing how to integrate with a web application""" + print("\n🌐 Web Integration Example") + print("=" * 50) + + # Simulate a web application using the API + class MockWebApp: + def __init__(self): + self.manager = BDFRManager("./web_downloads") + self.active_sessions = {} + + async def handle_download_request(self, user_id: str, subreddit: str, limit: int): + """Handle a download request from the web interface""" + + # Create custom callback for this user + callback = WebSocketCallback(f"ws_{user_id}") + + # Create and start download + download_id = self.manager.download_subreddit( + subreddit, + limit=limit, + progress_callbacks=[callback] + ) + + # Track for this user session + if user_id not in self.active_sessions: + self.active_sessions[user_id] = [] + self.active_sessions[user_id].append(download_id) + + return { + "success": True, + "download_id": download_id, + "message": f"Started download of r/{subreddit} (limit: {limit})" + } + + async def get_user_downloads(self, user_id: str): + """Get all downloads for a user""" + if user_id not in self.active_sessions: + return [] + + downloads = [] + for download_id in self.active_sessions[user_id]: + status = self.manager.get_download_status(download_id) + if status: + downloads.append(status) + + return downloads + + async def cancel_user_download(self, user_id: str, download_id: str): + """Cancel a specific download for a user""" + if user_id in self.active_sessions and download_id in self.active_sessions[user_id]: + success = self.manager.cancel_download(download_id) + if success: + self.active_sessions[user_id].remove(download_id) + return {"success": True, "message": "Download cancelled"} + else: + return {"success": False, "message": "Failed to cancel download"} + + return {"success": False, "message": "Download not found for user"} + + # Simulate web app usage + app = MockWebApp() + + # Simulate user requests + user_id = "user123" + + # User starts a download + result1 = await app.handle_download_request(user_id, "technology", 20) + print(f"User request result: {result1}") + + # User starts another download + result2 = await app.handle_download_request(user_id, "science", 15) + print(f"User request result: {result2}") + + # Check user's downloads + user_downloads = await app.get_user_downloads(user_id) + print(f"User has {len(user_downloads)} active downloads:") + for download in user_downloads: + print(f" - {download['id']}: {download['status']} ({download['progress']:.1f}%)") + + # Cancel one download + if user_downloads: + cancel_result = await app.cancel_user_download(user_id, user_downloads[0]['id']) + print(f"Cancel result: {cancel_result}") + + return len(user_downloads) + + +async def example_error_handling(): + """Example of error handling""" + print("\n⚠️ Error Handling Example") + print("=" * 50) + + manager = BDFRManager("./test_downloads") + + # Try to download from a non-existent subreddit + download_id = manager.download_subreddit( + "this_subreddit_does_not_exist", + limit=5 + ) + + print(f"📋 Created download for non-existent subreddit: {download_id}") + + # Monitor for error + for _ in range(5): # Check for up to 10 seconds + status = manager.get_download_status(download_id) + if not status: + print("❌ Download disappeared") + break + + print(f"📊 Status: {status['status']}") + + if status['status'] == 'failed': + print(f"🏁 Download failed as expected: {status.get('error', 'Unknown error')}") + break + + await asyncio.sleep(2) + + return download_id + + +async def main(): + """Run all examples""" + print("🎯 BDFR API Examples") + print("=" * 60) + print("This demonstrates the new BDFR API layer for direct web integration") + print() + + try: + # Run basic example + await example_basic_usage() + + # Run advanced example + await example_advanced_usage() + + # Run user download example + await example_user_download() + + # Run archive example + await example_archive_operation() + + # Run web integration example + await example_web_integration() + + # Run error handling example + await example_error_handling() + + print("\n🎉 All examples completed!") + + except KeyboardInterrupt: + print("\n⏹️ Examples interrupted by user") + except Exception as e: + print(f"\n❌ Error running examples: {e}") + import traceback + traceback.print_exc() + + +if __name__ == "__main__": + # Run the examples + asyncio.run(main()) \ No newline at end of file diff --git a/tests/test_file_locking_fix.py b/tests/test_file_locking_fix.py new file mode 100644 index 0000000..e86f58e --- /dev/null +++ b/tests/test_file_locking_fix.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +""" +Test script to verify that the file locking issue is fixed. +This script simulates the scenario where a download fails and then tries to redownload. +""" + +import asyncio +import logging +import os +import tempfile +import time +from pathlib import Path + +# Add the bdfr module to the path +import sys +sys.path.insert(0, str(Path(__file__).parent)) + +from bdfr.api import BDFRManager, DownloadType, LoggingCallback, ProgressEvent + +class TestProgressCallback(LoggingCallback): + """Test callback that simulates a failure""" + + def __init__(self): + super().__init__("test_logger") + self.events = [] + + async def on_progress(self, event: ProgressEvent): + self.events.append(event) + await super().on_progress(event) + + async def on_error(self, event: ProgressEvent): + self.events.append(event) + await super().on_error(event) + + async def on_completed(self, event: ProgressEvent): + self.events.append(event) + await super().on_completed(event) + +def test_file_locking_fix(): + """Test that the file locking issue is resolved""" + print("Testing file locking fix...") + + # Create a temporary directory for testing + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + print(f"Using temporary directory: {temp_path}") + + # Create BDFR manager + manager = BDFRManager(temp_path) + + # Test 1: Create a download that will fail + print("\n1. Creating first download (will fail)...") + download_id1 = manager.create_download( + DownloadType.USER, + "test_user_12345", # This user doesn't exist, should fail + progress_callbacks=[TestProgressCallback()] + ) + + # Start the download (it should fail) + manager.start_download(download_id1) + + # Wait a bit for the download to start and fail + time.sleep(2) + + # Check status + status1 = manager.get_download_status(download_id1) + print(f"First download status: {status1['status'] if status1 else 'Not found'}") + + # Test 2: Try to create a second download immediately after + print("\n2. Creating second download (should work without file locking error)...") + download_id2 = manager.create_download( + DownloadType.USER, + "test_user_67890", # This user also doesn't exist, should fail + progress_callbacks=[TestProgressCallback()] + ) + + # Start the second download + success = manager.start_download(download_id2) + + if success: + print("SUCCESS: Second download started successfully (no file locking error)") + else: + print("FAILED: Failed to start second download") + return False + + # Wait for second download to fail + time.sleep(2) + + # Check status + status2 = manager.get_download_status(download_id2) + print(f"Second download status: {status2['status'] if status2 else 'Not found'}") + + # Test 3: Check that log files are unique + print("\n3. Checking for unique log files...") + logs_dir = temp_path / "logs" + if logs_dir.exists(): + log_files = list(logs_dir.glob("*.log")) + print(f"Found {len(log_files)} log files:") + for log_file in log_files: + print(f" - {log_file.name}") + # Check if file is accessible (not locked) + try: + with open(log_file, 'r') as f: + content = f.read() + print(f" SUCCESS: Log file is accessible ({len(content)} characters)") + except PermissionError: + print(f" FAILED: Log file is still locked!") + return False + else: + print("No logs directory found") + + # Test 4: Try to create a third download to ensure cleanup worked + print("\n4. Creating third download to verify cleanup...") + download_id3 = manager.create_download( + DownloadType.USER, + "test_user_cleanup", + progress_callbacks=[TestProgressCallback()] + ) + + success3 = manager.start_download(download_id3) + if success3: + print("SUCCESS: Third download started successfully (cleanup worked)") + else: + print("FAILED: Third download failed to start") + return False + + # Wait and check final status + time.sleep(2) + status3 = manager.get_download_status(download_id3) + print(f"Third download status: {status3['status'] if status3 else 'Not found'}") + + print("\nSUCCESS: All tests passed! File locking issue appears to be fixed.") + return True + +if __name__ == "__main__": + try: + success = test_file_locking_fix() + if success: + print("\nTest completed successfully!") + sys.exit(0) + else: + print("\nTest failed!") + sys.exit(1) + except Exception as e: + print(f"\nTest failed with exception: {e}") + import traceback + traceback.print_exc() + sys.exit(1) \ No newline at end of file diff --git a/tests/test_user_folder_structure.py b/tests/test_user_folder_structure.py new file mode 100644 index 0000000..f0b0ed8 --- /dev/null +++ b/tests/test_user_folder_structure.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Test script to verify user folder structure changes. + +This script tests that: +1. Subreddit downloads go to: downloads/subreddit_name/files +2. User downloads go to: downloads/username/subreddit_name/files +""" + +import sys +import tempfile +from pathlib import Path + +# Set UTF-8 encoding for Windows console +if sys.platform == 'win32': + import codecs + sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer, 'strict') + sys.stderr = codecs.getwriter('utf-8')(sys.stderr.buffer, 'strict') + +from bdfr.api import BDFRManager, DownloadType +from bdfr.configuration import Configuration + + +def test_subreddit_directory_structure(): + """Test that subreddit downloads use correct directory structure""" + print("\n=== Testing Subreddit Directory Structure ===") + + with tempfile.TemporaryDirectory() as tmpdir: + manager = BDFRManager(download_directory=tmpdir) + + # Create a download for a subreddit + download_id = manager.create_download( + DownloadType.SUBREDDIT, + "test_subreddit" + ) + + download_info = manager.get_download_status(download_id) + config = download_info["config"] + + expected_dir = str(Path(tmpdir)) + actual_dir = config.directory + + print(f"Expected directory: {expected_dir}") + print(f"Actual directory: {actual_dir}") + print(f"Subreddit config: {config.subreddit}") + print(f"Folder scheme: {config.folder_scheme}") + + assert actual_dir == expected_dir, f"Subreddit directory mismatch!" + print("[OK] Subreddit directory structure is correct") + print(f" Files will be saved to: {actual_dir}/{{SUBREDDIT}}/{{files}}") + + +def test_user_directory_structure(): + """Test that user downloads use correct directory structure""" + print("\n=== Testing User Directory Structure ===") + + with tempfile.TemporaryDirectory() as tmpdir: + manager = BDFRManager(download_directory=tmpdir) + + # Create a download for a user + username = "test_user" + download_id = manager.create_download( + DownloadType.USER, + username + ) + + download_info = manager.get_download_status(download_id) + config = download_info["config"] + + expected_dir = str(Path(tmpdir) / username) + actual_dir = config.directory + + print(f"Expected directory: {expected_dir}") + print(f"Actual directory: {actual_dir}") + print(f"User config: {config.user}") + print(f"Folder scheme: {config.folder_scheme}") + + assert actual_dir == expected_dir, f"User directory mismatch!" + print("[OK] User directory structure is correct") + print(f" Files will be saved to: {actual_dir}/{{SUBREDDIT}}/{{files}}") + + +def test_convenience_method(): + """Test the convenience method download_user()""" + print("\n=== Testing download_user() Convenience Method ===") + + with tempfile.TemporaryDirectory() as tmpdir: + manager = BDFRManager(download_directory=tmpdir) + + # Don't actually start the download, just check the config + username = "convenience_test_user" + + # Create config manually like the convenience method does + config = Configuration() + user_directory = Path(tmpdir) / username + config.directory = str(user_directory) + config.user = [username] + + expected_dir = str(Path(tmpdir) / username) + actual_dir = config.directory + + print(f"Expected directory: {expected_dir}") + print(f"Actual directory: {actual_dir}") + print(f"User config: {config.user}") + + assert actual_dir == expected_dir, f"Convenience method directory mismatch!" + print("[OK] Convenience method directory structure is correct") + + +def demonstrate_folder_structure(): + """Demonstrate the folder structure for both download types""" + print("\n=== Folder Structure Demonstration ===") + print("\nWhen downloading from a SUBREDDIT 'python':") + print(" downloads/") + print(" └── python/") + print(" ├── file1.jpg") + print(" ├── file2.png") + print(" └── file3.mp4") + + print("\nWhen downloading from a USER 'spez' who posts to multiple subreddits:") + print(" downloads/") + print(" └── spez/") + print(" ├── python/") + print(" │ ├── file1.jpg") + print(" │ └── file2.png") + print(" ├── announcements/") + print(" │ └── file3.jpg") + print(" └── pics/") + print(" └── file4.png") + + print("\n[OK] This structure allows:") + print(" 1. Easy identification of user-specific downloads") + print(" 2. Organization by subreddit within each user folder") + print(" 3. No conflicts between subreddit and user downloads") + + +if __name__ == "__main__": + print("=" * 60) + print("Testing User Folder Structure Changes") + print("=" * 60) + + try: + test_subreddit_directory_structure() + test_user_directory_structure() + test_convenience_method() + demonstrate_folder_structure() + + print("\n" + "=" * 60) + print("[SUCCESS] All tests passed!") + print("=" * 60) + + except AssertionError as e: + print(f"\n[FAIL] Test failed: {e}") + exit(1) + except Exception as e: + print(f"\n[ERROR] Unexpected error: {e}") + import traceback + traceback.print_exc() + exit(1) \ No newline at end of file diff --git a/web_interface/.dockerignore b/web_interface/.dockerignore new file mode 100644 index 0000000..8cce7f4 --- /dev/null +++ b/web_interface/.dockerignore @@ -0,0 +1,35 @@ +__pycache__ +*.pyc +*.pyo +*.pyd +.Python +env +pip-log.txt +pip-delete-this-directory.txt +.tox +.coverage +.coverage.* +.pytest_cache +nosetests.xml +coverage.xml +*.cover +*.log +.git +.mypy_cache +.pytest_cache +.hypothesis +*.egg-info/ +dist/ +build/ +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +.DS_Store +Thumbs.db +*.swp +*.swo +*~ \ No newline at end of file diff --git a/web_interface/.env.example b/web_interface/.env.example new file mode 100644 index 0000000..40c46e8 --- /dev/null +++ b/web_interface/.env.example @@ -0,0 +1,17 @@ +# BDFR Web Interface Configuration +# Copy this file to .env and update the values as needed + +# Reddit OAuth Configuration +# You MUST set this to match your Reddit OAuth app settings +# Go to https://www.reddit.com/prefs/apps, create/edit your app, and use the exact redirect URI +BDFR_REDIRECT_URI=http://localhost:8000/auth/callback + +# OAuth Credentials (from your Reddit OAuth app) +# Get these from: https://www.reddit.com/prefs/apps +BDFR_CLIENT_ID=your_client_id_here +BDFR_CLIENT_SECRET=your_client_secret_here + +# Server Configuration (optional) +# HOST=0.0.0.0 +# PORT=8000 +# DEBUG=true \ No newline at end of file diff --git a/web_interface/Dockerfile b/web_interface/Dockerfile new file mode 100644 index 0000000..e571277 --- /dev/null +++ b/web_interface/Dockerfile @@ -0,0 +1,35 @@ +FROM python:3.11-slim + +# Set working directory +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + gcc \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements first for better caching +COPY requirements.txt . + +# Install Python dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY app/ ./app/ +COPY templates/ ./templates/ +COPY static/ ./static/ + +# Create non-root user +RUN useradd --create-home --shell /bin/bash app \ + && chown -R app:app /app +USER app + +# Expose port +EXPOSE 8000 + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD curl -f http://localhost:8000/health || exit 1 + +# Run the application +CMD ["python", "/app/app/main.py"] \ No newline at end of file diff --git a/web_interface/README.md b/web_interface/README.md new file mode 100644 index 0000000..6328665 --- /dev/null +++ b/web_interface/README.md @@ -0,0 +1,202 @@ +# BDFR Web Interface + +A modern web interface for the Bulk Downloader for Reddit (BDFR) built with FastAPI, WebSockets, and vanilla JavaScript. + +## Features + +- **Modern UI**: Clean, responsive design with gradient backgrounds and smooth animations +- **Real-time Progress**: WebSocket-based progress updates for active downloads +- **Subreddit Downloads**: Download posts from any subreddit with customizable limits and sorting +- **User Downloads**: Download posts from specific users +- **Status Monitoring**: Real-time system status and connection monitoring +- **Form Validation**: Client-side validation with visual feedback +- **Error Handling**: Comprehensive error handling with user-friendly notifications + +## Project Structure + +``` +web_interface/ +├── app/ +│ └── main.py # FastAPI application +├── static/ +│ ├── css/ +│ │ └── style.css # Modern CSS styling +│ └── js/ +│ └── app.js # WebSocket client and form handling +├── templates/ +│ └── index.html # Main web interface +└── requirements.txt # Python dependencies +``` + +## Installation + +1. **Install Dependencies**: + ```bash + cd web_interface + pip install -r requirements.txt + ``` + +2. **Run the Application**: + ```bash + cd app + python main.py + ``` + +3. **Access the Interface**: + Open your browser and navigate to `http://localhost:8000` + +## API Endpoints + +### Download Endpoints +- `POST /api/download/subreddit` - Start subreddit download +- `POST /api/download/user` - Start user download +- `GET /api/downloads` - List all active downloads +- `GET /api/downloads/{download_id}` - Get specific download status +- `DELETE /api/downloads/{download_id}` - Cancel download + +### WebSocket +- `ws://localhost:8000/ws/progress` - Real-time progress updates + +### Status Endpoints +- `GET /` - Main web interface +- `GET /health` - Health check +- `GET /api/bdfr/status` - BDFR system status + +## Configuration + +The application uses the following default settings: +- **Host**: `0.0.0.0` +- **Port**: `8000` +- **WebSocket Path**: `/ws/progress` +- **Static Files**: Served from `/static` + +## Docker Support + +To run with Docker: + +```bash +# Build the image +docker build -t bdfr-web-interface . + +# Run the container +docker run -p 8000:8000 bdfr-web-interface +``` + +## Development + +### Adding New Features + +1. **Backend Changes**: Modify `app/main.py` to add new endpoints +2. **Frontend Changes**: Update `templates/index.html` for UI changes +3. **Styling**: Modify `static/css/style.css` for visual changes +4. **JavaScript**: Update `static/js/app.js` for client-side functionality + +### WebSocket Integration + +The WebSocket connection automatically handles: +- Connection establishment and reconnection +- Progress updates from the server +- Error handling and user notifications +- Real-time UI updates + +### Form Handling + +Both download forms include: +- Input validation +- Loading states +- Success/error notifications +- Automatic form reset on success + +## Integration with BDFR + +### Direct BDFR API Integration + +This interface uses the direct BDFR API integration, eliminating the need for subprocess console parsing: + +- `/api/download/subreddit` - Downloads from subreddits using `BDFRManager.download_subreddit()` +- `/api/download/user` - Downloads from users using `BDFRManager.download_user()` +- `/api/bdfr/status` - Returns BDFR system status and capabilities +- `/ws/progress` - Provides real-time progress updates via WebSocket + +The web interface imports `BDFRManager` directly from `bdfr.api` and uses structured progress callbacks for seamless integration. + +### Migration Notes + +**Previous Approach (Subprocess-based)**: +- Used `subprocess.Popen` to start BDFR CLI +- Parsed console output with regex for progress updates +- Required `BDFRRunner` class for process management +- Used `threading.Thread` and `queue.Queue` for coordination + +**Current Approach (Direct API)**: +- Direct integration with `BDFRManager` from `bdfr.api` +- Structured `ProgressEvent` callbacks instead of console parsing +- Thread-safe progress tracking with `ProgressCallback` interface +- No subprocess overhead or console output parsing required + +The migration provides better error handling, structured progress events, and eliminates console parsing complexity. + +## Browser Support + +- Modern browsers with WebSocket support +- Chrome 60+ +- Firefox 55+ +- Safari 11+ +- Edge 79+ + +## Security Considerations + +- CORS is enabled for all origins (configure for production) +- Input validation on both client and server +- No authentication implemented (add as needed) +- WebSocket connections are not secured (use WSS in production) + +## Production Deployment + +For production deployment: + +1. Configure CORS for specific origins +2. Add authentication/authorization +3. Use HTTPS/WSS for secure connections +4. Configure proper logging +5. Set up reverse proxy (nginx recommended) +6. Add rate limiting +7. Configure environment variables + +## Troubleshooting + +### Common Issues + +1. **WebSocket Connection Failed**: + - Check if the server is running + - Verify firewall settings + - Check browser console for errors + +2. **Downloads Not Starting**: + - Verify BDFR integration is configured + - Check server logs for errors + - Ensure form data is valid + +3. **Static Files Not Loading**: + - Verify static file paths + - Check file permissions + - Ensure proper MIME types + +### Debug Mode + +Run with debug logging: +```bash +python main.py --log-level debug +``` + +## Contributing + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Test thoroughly +5. Submit a pull request + +## License + +This project is part of the BDFR ecosystem. See the main project license for details. \ No newline at end of file diff --git a/web_interface/STARTUP.md b/web_interface/STARTUP.md new file mode 100644 index 0000000..658a399 --- /dev/null +++ b/web_interface/STARTUP.md @@ -0,0 +1,177 @@ +# BDFR Web Interface Startup Scripts + +This directory contains simple startup scripts to easily run the BDFR web interface application. + +## Available Scripts + +### 🚀 Quick Start + +Choose the appropriate script for your operating system: + +- **`start.py`** - Cross-platform Python script (recommended) +- **`start.sh`** - Unix/Linux/macOS shell script +- **`start.bat`** - Windows batch script + +## Usage + +### Option 1: Python Script (Cross-platform) + +```bash +# Navigate to the web_interface directory +cd web_interface + +# Run the startup script +python start.py +``` + +### Option 2: Shell Script (Unix/Linux/macOS) + +```bash +# Navigate to the web_interface directory +cd web_interface + +# Make sure the script is executable +chmod +x start.sh + +# Run the startup script +./start.sh +``` + +### Option 3: Batch Script (Windows) + +```cmd +REM Navigate to the web_interface directory +cd web_interface + +REM Run the startup script +start.bat +``` + +## What the Scripts Do + +1. **Check Python version** - Ensures Python 3.8+ is installed +2. **Install dependencies** - Automatically installs required packages from `requirements.txt` +3. **Verify BDFR module** - Checks if the BDFR module is available +4. **Start the server** - Launches the FastAPI application with uvicorn + +## Server Information + +Once started, the web interface will be available at: +- **Main interface**: http://localhost:8000 +- **API documentation**: http://localhost:8000/docs +- **Health check**: http://localhost:8000/health + +## Features + +- ✅ Automatic dependency management +- ✅ Cross-platform compatibility +- ✅ Colored output for better user experience +- ✅ Error handling and informative messages +- ✅ Graceful server shutdown +- ✅ BDFR module availability checking + +## Requirements + +- Python 3.8 or higher +- Internet connection (for installing dependencies) +- BDFR module in Python path (parent directory should contain the BDFR package) +- Reddit OAuth application (for authentication features) + +## Reddit OAuth Setup + +To use the authentication features, you need to: + +1. **Create a Reddit OAuth Application**: + - Go to [Reddit App Preferences](https://www.reddit.com/prefs/apps) + - Click "Create App" or "Create Another App" + - Choose "web app" as the application type + - Set a name (e.g., "BDFR Web Interface") + - Set redirect URI to: `http://localhost:8000/auth/callback` + +2. **Configure the Redirect URI** (if using a different port or domain): + - Run the OAuth setup helper: `python setup_oauth.py` + - Or manually copy `.env.example` to `.env` + - Update the following in the `.env` file: + - `BDFR_REDIRECT_URI` - Your OAuth redirect URI + - `BDFR_CLIENT_ID` - Your OAuth client ID (from Reddit app) + - `BDFR_CLIENT_SECRET` - Your OAuth client secret (from Reddit app) + - Make sure the redirect URI matches exactly what you set in your Reddit OAuth app + +3. **Update BDFR Configuration**: + - The web interface uses the same OAuth credentials as BDFR + - You need to either update your existing Reddit OAuth app or create a new one + +## Option A: Update Existing Reddit OAuth App + +If you want to use the same OAuth app for both BDFR CLI and web interface: + +1. Go to [Reddit App Preferences](https://www.reddit.com/prefs/apps) +2. Find your existing app (the one with client ID `U-6gk4ZCh3IeNQ`) +3. Click "edit" and add your redirect URI to the "redirect uris" field: + - `http://localhost:8000/auth/callback` +4. Save the changes + +## Option B: Create a New Reddit OAuth App (Recommended) + +For better separation between CLI and web interface: + +1. Go to [Reddit App Preferences](https://www.reddit.com/prefs/apps) +2. Click "Create App" or "Create Another App" +3. Fill in the details: + - **Name**: `BDFR Web Interface` (or your preferred name) + - **App type**: `web app` + - **Description**: `Web interface for BDFR (Bulk Downloader for Reddit)` + - **About URL**: (optional) + - **Redirect URI**: `http://localhost:8000/auth/callback` +4. Click "Create app" +5. Copy the client ID and client secret +6. Update `bdfr/default_config.cfg` with the new credentials: + ``` + client_id = YOUR_NEW_CLIENT_ID + client_secret = YOUR_NEW_CLIENT_SECRET + ``` + +## Troubleshooting + +### "BDFR module not found" +Make sure you're running the script from the correct directory, or ensure the parent directory containing the BDFR package is in your Python path. + +### "Python 3.8+ required" +Install Python 3.8 or higher from the official Python website. + +### "Permission denied" (Unix/Linux/macOS) +Make sure the shell script has execute permissions: +```bash +chmod +x start.sh +``` + +### "invalid redirect_uri parameter" (OAuth Error) +This error occurs when the redirect URI doesn't match what you configured in your Reddit OAuth app: + +1. **Verify your Reddit OAuth app settings**: + - Go to [Reddit App Preferences](https://www.reddit.com/prefs/apps) + - Find your app and check the redirect URI + - Make sure it exactly matches what you're using + +2. **Update the redirect URI**: + - Copy `.env.example` to `.env` + - Set `BDFR_REDIRECT_URI` to match your Reddit OAuth app + - Example: `BDFR_REDIRECT_URI=http://localhost:8000/auth/callback` + +3. **Common redirect URI formats**: + - Local development: `http://localhost:8000/auth/callback` + - With custom port: `http://localhost:3000/auth/callback` + - Production: `https://yourdomain.com/auth/callback` + +4. **Recreate your OAuth app if needed**: + - Delete the existing app in Reddit + - Create a new one with the correct redirect URI + +## Manual Alternative + +If you prefer to run the server manually: + +```bash +cd web_interface +pip install -r requirements.txt +python -m uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload \ No newline at end of file diff --git a/web_interface/app/auth.py b/web_interface/app/auth.py new file mode 100644 index 0000000..f21c53a --- /dev/null +++ b/web_interface/app/auth.py @@ -0,0 +1,311 @@ +""" +OAuth2 Authentication module for BDFR Web Interface + +This module handles OAuth2 authentication flow for the web interface, +integrating with BDFR's existing OAuth2 system. +""" + +import asyncio +import json +import logging +import secrets +import time +from datetime import datetime, timedelta +from typing import Dict, Optional, Any +from urllib.parse import urlencode + +import httpx +from fastapi import HTTPException, status + +# Try to import BDFR modules, but handle gracefully if not available +try: + from bdfr.oauth2 import OAuth2Authenticator, OAuth2TokenManager + from bdfr.exceptions import RedditAuthenticationError + BDFR_AVAILABLE = True +except ImportError: + BDFR_AVAILABLE = False + # Create mock classes for when BDFR is not available + class OAuth2Authenticator: + pass + class OAuth2TokenManager: + pass + class RedditAuthenticationError(Exception): + pass + +logger = logging.getLogger(__name__) + + +class WebOAuth2Manager: + """OAuth2 manager for web interface authentication""" + + def __init__(self, client_id: str, client_secret: str, scopes: list = None): + self.client_id = client_id + self.client_secret = client_secret + self.scopes = scopes or ["identity", "history", "read", "save", "mysubreddits"] + + # In-memory storage for OAuth2 states and tokens + # In production, this should be replaced with a proper database + self.oauth_states = {} + self.refresh_tokens = {} + self.access_tokens = {} + # Store Reddit usernames per session state + self.usernames = {} + + # Reddit OAuth2 endpoints + self.reddit_auth_url = "https://www.reddit.com/api/v1/authorize" + self.reddit_token_url = "https://www.reddit.com/api/v1/access_token" + self.reddit_user_info_url = "https://oauth.reddit.com/api/v1/me" + + # Token expiration tracking + self.token_expiry = {} + + def generate_state(self) -> str: + """Generate a secure random state for OAuth2""" + state = secrets.token_urlsafe(32) + self.oauth_states[state] = { + "created_at": time.time(), + "used": False + } + return state + + def validate_state(self, state: str) -> bool: + """Validate OAuth2 state parameter""" + if state not in self.oauth_states: + return False + + state_data = self.oauth_states[state] + if state_data["used"]: + return False + + # States expire after 10 minutes + if time.time() - state_data["created_at"] > 600: + del self.oauth_states[state] + return False + + return True + + def mark_state_used(self, state: str): + """Mark OAuth2 state as used""" + if state in self.oauth_states: + self.oauth_states[state]["used"] = True + + def get_authorization_url(self, redirect_uri: str) -> Dict[str, str]: + """Generate OAuth2 authorization URL""" + state = self.generate_state() + + params = { + "client_id": self.client_id, + "response_type": "code", + "state": state, + "redirect_uri": redirect_uri, + "scope": " ".join(self.scopes), + "duration": "permanent" + } + + auth_url = f"{self.reddit_auth_url}?{urlencode(params)}" + + return { + "authorization_url": auth_url, + "state": state + } + + async def exchange_code_for_token(self, code: str, state: str, redirect_uri: str) -> Dict[str, Any]: + """Exchange authorization code for access token""" + if not self.validate_state(state): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid or expired state parameter" + ) + + self.mark_state_used(state) + + data = { + "grant_type": "authorization_code", + "code": code, + "redirect_uri": redirect_uri + } + + headers = { + "User-Agent": "BDFR-Web-Interface/1.0" + } + + # Use HTTP Basic Auth for client credentials + auth = (self.client_id, self.client_secret) + + async with httpx.AsyncClient() as client: + try: + response = await client.post( + self.reddit_token_url, + data=data, + auth=auth, + headers=headers, + timeout=30.0 + ) + + if response.status_code != 200: + error_detail = response.json() if response.headers.get("content-type", "").startswith("application/json") else response.text + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Token exchange failed: {error_detail}" + ) + + token_data = response.json() + + # Store tokens + access_token = token_data["access_token"] + refresh_token = token_data.get("refresh_token") + + if refresh_token: + self.refresh_tokens[state] = refresh_token + self.access_tokens[state] = access_token + + # Set expiry (Reddit tokens typically last 1 hour) + self.token_expiry[state] = time.time() + token_data.get("expires_in", 3600) + + # Attempt to fetch and store the Reddit username for this session + username = None + try: + user_info = await self.get_user_info(access_token) + username = user_info.get("name") + except Exception as e: + logger.warning(f"Failed to fetch user info during token exchange: {e}") + + if username: + self.usernames[state] = username + + return { + "access_token": access_token, + "refresh_token": refresh_token, + "expires_in": token_data.get("expires_in", 3600), + "token_type": token_data.get("token_type", "bearer"), + "state": state, + "username": username + } + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="No refresh token received" + ) + + except httpx.TimeoutException: + raise HTTPException( + status_code=status.HTTP_408_REQUEST_TIMEOUT, + detail="Token exchange timed out" + ) + except Exception as e: + logger.error(f"Token exchange error: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal server error during token exchange" + ) + + async def get_user_info(self, access_token: str) -> Dict[str, Any]: + """Get user information using access token""" + headers = { + "Authorization": f"Bearer {access_token}", + "User-Agent": "BDFR-Web-Interface/1.0" + } + + async with httpx.AsyncClient() as client: + try: + response = await client.get( + self.reddit_user_info_url, + headers=headers, + timeout=30.0 + ) + + if response.status_code != 200: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid access token" + ) + + return response.json() + + except httpx.TimeoutException: + raise HTTPException( + status_code=status.HTTP_408_REQUEST_TIMEOUT, + detail="User info request timed out" + ) + except Exception as e: + logger.error(f"User info error: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Error retrieving user information" + ) + + def is_token_expired(self, state: str) -> bool: + """Check if access token is expired""" + if state not in self.token_expiry: + return True + return time.time() > self.token_expiry[state] + + def get_valid_token(self, state: str) -> Optional[str]: + """Get valid access token, refreshing if necessary""" + if state not in self.access_tokens: + return None + + if self.is_token_expired(state): + # Token expired, would need refresh logic here + # For now, just return None to indicate re-auth needed + return None + + return self.access_tokens[state] + + def revoke_session(self, state: str): + """Revoke OAuth2 session""" + if state in self.oauth_states: + del self.oauth_states[state] + if state in self.refresh_tokens: + del self.refresh_tokens[state] + if state in self.access_tokens: + del self.access_tokens[state] + if state in self.token_expiry: + del self.token_expiry[state] + if state in self.usernames: + del self.usernames[state] + + def get_auth_status(self, state: str = None) -> Dict[str, Any]: + """Get authentication status""" + if not state: + return { + "authenticated": False, + "message": "No active session" + } + + if state not in self.access_tokens: + return { + "authenticated": False, + "message": "No tokens found for session" + } + + access_token = self.get_valid_token(state) + if not access_token: + return { + "authenticated": False, + "message": "Token expired or invalid" + } + + return { + "authenticated": True, + "expires_at": self.token_expiry.get(state, 0), + "scopes": self.scopes, + "username": self.usernames.get(state) + } + + +# Global OAuth2 manager instance +oauth_manager = None + + +def init_oauth_manager(client_id: str, client_secret: str, scopes: list = None): + """Initialize the global OAuth2 manager""" + global oauth_manager + oauth_manager = WebOAuth2Manager(client_id, client_secret, scopes) + + +def get_oauth_manager() -> WebOAuth2Manager: + """Get the global OAuth2 manager instance""" + if oauth_manager is None: + raise RuntimeError("OAuth2 manager not initialized") + return oauth_manager \ No newline at end of file diff --git a/web_interface/app/main.py b/web_interface/app/main.py new file mode 100644 index 0000000..02cb65a --- /dev/null +++ b/web_interface/app/main.py @@ -0,0 +1,960 @@ +from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, Form, File, UploadFile, Query, status +from fastapi.responses import HTMLResponse, FileResponse, RedirectResponse +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates +from fastapi.middleware.cors import CORSMiddleware +from fastapi import Request +import json +import asyncio +import os +import configparser +import logging + +# Load environment variables from .env file if available +try: + from dotenv import load_dotenv + load_dotenv() +except ImportError: + # python-dotenv not installed, use os.environ directly + pass +from typing import List, Dict, Any, Optional +from datetime import datetime +from urllib.parse import urlencode + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Import authentication module +from .auth import init_oauth_manager, get_oauth_manager + +# Import BDFR API layer +import sys +import os +# Add the parent directory (BDFR root) to Python path +bdfr_root = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 'bdfr') +sys.path.insert(0, bdfr_root) + +# Also add the current bdfr directory to path +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), '')) + +try: + from bdfr.api import BDFRManager, ProgressEvent, ProgressCallback, get_bdfr_manager + BDFR_AVAILABLE = True +except ImportError as e: + logger.warning(f"Failed to import BDFR API: {e}. BDFR modules may not be available.") + BDFR_AVAILABLE = False + # Create a mock BDFRManager for when BDFR is not available + class MockBDFRManager: + def __init__(self, *args, **kwargs): + pass + def download_subreddit(self, *args, **kwargs): + raise NotImplementedError("BDFR not available") + def download_user(self, *args, **kwargs): + raise NotImplementedError("BDFR not available") + def get_download_status(self, *args, **kwargs): + return None + def cancel_download(self, *args, **kwargs): + return False + + class MockProgressCallback: + pass + + class MockProgressEvent: + def __init__(self, *args, **kwargs): + pass + + BDFRManager = MockBDFRManager + ProgressEvent = MockProgressEvent + ProgressCallback = MockProgressCallback + def get_bdfr_manager(*args, **kwargs): + return MockBDFRManager() + +app = FastAPI(title="BDFR Web Interface", version="1.0.0") + +# Initialize OAuth2 manager +def init_oauth(): + """Initialize OAuth2 manager with credentials from environment or BDFR config""" + try: + # Try to get credentials from environment variables first + client_id = os.getenv("BDFR_CLIENT_ID") + client_secret = os.getenv("BDFR_CLIENT_SECRET") + + if client_id and client_secret: + # Use environment credentials + logger.info("Using OAuth credentials from environment variables") + scopes = ["identity", "history", "read", "save", "mysubreddits"] + else: + # Fall back to BDFR config file + logger.info("Using OAuth credentials from BDFR config file") + config = configparser.ConfigParser() + config.read("../bdfr/default_config.cfg") + + client_id = config.get("DEFAULT", "client_id") + client_secret = config.get("DEFAULT", "client_secret") + scopes_str = config.get("DEFAULT", "scopes", fallback="identity,read") + + # Parse scopes + scopes = [scope.strip() for scope in scopes_str.split(",")] + + init_oauth_manager(client_id, client_secret, scopes) + logger.info("OAuth2 manager initialized successfully") + except Exception as e: + logger.error(f"Failed to initialize OAuth2 manager: {e}") + # Use default credentials if config fails + init_oauth_manager("U-6gk4ZCh3IeNQ", "7CZHY6AmKweZME5s50SfDGylaPg") + +# Initialize OAuth2 on startup +# Use a configurable redirect URI - this should match your Reddit OAuth app settings +redirect_uri = os.getenv("BDFR_REDIRECT_URI", "http://localhost:8000/auth/callback") +init_oauth() + +# Enable CORS +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# Mount static files +import os +current_dir = os.path.dirname(os.path.abspath(__file__)) +static_dir = os.path.join(current_dir, "..", "static") +template_dir = os.path.join(current_dir, "..", "templates") + +# Ensure directories exist +os.makedirs(static_dir, exist_ok=True) +os.makedirs(template_dir, exist_ok=True) + +app.mount("/static", StaticFiles(directory=static_dir), name="static") +templates = Jinja2Templates(directory=template_dir) + +# WebSocket connection manager +class ConnectionManager: + def __init__(self): + self.active_connections: List[WebSocket] = [] + + async def connect(self, websocket: WebSocket): + logger.info(f"[WEBSOCKET] WebSocket connection attempt from {websocket.client}") + await websocket.accept() + self.active_connections.append(websocket) + logger.info(f"[WEBSOCKET] New connection established from {websocket.client}. Total connections: {len(self.active_connections)}") + + def disconnect(self, websocket: WebSocket): + if websocket in self.active_connections: + self.active_connections.remove(websocket) + logger.info(f"[WEBSOCKET] Connection disconnected from {websocket.client}. Total connections: {len(self.active_connections)}") + + async def send_personal_message(self, message: str, websocket: WebSocket): + try: + await websocket.send_text(message) + except Exception as e: + logger.warning(f"Failed to send personal message: {e}") + self.disconnect(websocket) + + async def broadcast(self, message: str): + """Broadcast message to all active connections - simplified version""" + logger.debug(f"[WEBSOCKET] Broadcasting message to {len(self.active_connections)} connections") + + if not self.active_connections: + logger.debug("[WEBSOCKET] No active connections to broadcast to!") + return + + # Simple approach: try to send to all connections, remove failed ones immediately + alive_connections = [] + + for connection in self.active_connections: + try: + await connection.send_text(message) + alive_connections.append(connection) + logger.debug(f"[WEBSOCKET] Message sent successfully to connection {id(connection)}") + except Exception as e: + logger.warning(f"[WEBSOCKET] Removing failed connection {id(connection)}: {e}") + # Connection failed, don't add it to alive_connections + + # Update active connections to only include successful ones + self.active_connections = alive_connections + logger.debug(f"[WEBSOCKET] Broadcast complete. Active connections: {len(self.active_connections)}") + + async def is_connection_alive(self, websocket: WebSocket) -> bool: + """Check if a WebSocket connection is still alive""" + try: + # Try to send a ping frame (this is a WebSocket protocol ping) + await websocket.ping() + return True + except Exception: + return False + + async def cleanup_dead_connections(self): + """Simplified cleanup - just log the current state""" + logger.info(f"[WEBSOCKET] Cleanup check: {len(self.active_connections)} active connections") + +manager = ConnectionManager() + +# Initialize BDFR Manager +bdfr_manager = get_bdfr_manager("./downloads") + +# Store auth token for BDFR manager if available +_bdfr_auth_token = None + +# Active downloads tracking (now managed by BDFRManager, but kept for WebSocket compatibility) +active_downloads = {} + +# WebSocket-compatible progress callback +class WebSocketProgressCallback(ProgressCallback if BDFR_AVAILABLE else MockProgressCallback): + """Progress callback that sends updates to WebSocket clients""" + + def __init__(self, download_id: str, connection_manager: ConnectionManager): + self.download_id = download_id + self.connection_manager = connection_manager + + async def on_progress(self, event: ProgressEvent): + """Send progress update to WebSocket clients""" + logger.info(f"[WEBSOCKET-PROGRESS] Received progress event for download {event.download_id}: {event.message} ({event.progress}%)") + logger.info(f"[WEBSOCKET-PROGRESS] Active connections before broadcast: {len(self.connection_manager.active_connections)}") + try: + # Convert BDFR API event to web interface format + progress_data = { + "type": "progress", + "id": event.download_id, + "download_id": event.download_id, + "status": "running", + "message": event.message, + "progress": event.progress or 0, + "data": event.data, + "timestamp": event.timestamp.isoformat() + } + + # Update active_downloads for WebSocket compatibility + # Need to find the web interface download ID that corresponds to this BDFR download ID + web_download_id = None + for download_id, download_info in active_downloads.items(): + if download_info.get("bdfr_download_id") == event.download_id: + web_download_id = download_id + break + + if web_download_id: + # Keep server-side state in sync + active_downloads[web_download_id]["status"] = "running" + active_downloads[web_download_id]["progress"] = event.progress or 0 + if "items_processed" in event.data: + active_downloads[web_download_id]["items_processed"] = event.data["items_processed"] + if "items_found" in event.data: + active_downloads[web_download_id]["items_found"] = event.data["items_found"] + if "current_item" in event.data: + active_downloads[web_download_id]["current_item"] = event.data["current_item"] + if "phase" in event.data: + active_downloads[web_download_id]["phase"] = event.data["phase"] + + # Unify IDs for frontend to prevent duplicate cards + progress_data["id"] = web_download_id + progress_data["download_id"] = web_download_id + progress_data["bdfr_download_id"] = event.download_id + progress_data["web_download_id"] = web_download_id + progress_data["subreddit"] = active_downloads[web_download_id].get("subreddit") + progress_data["username"] = active_downloads[web_download_id].get("username") + progress_data["limit"] = active_downloads[web_download_id].get("limit") + else: + # No mapping yet; include bdfr id for debugging + progress_data["bdfr_download_id"] = event.download_id + progress_data["limit"] = (event.data or {}).get("limit") + + logger.info(f"[WEBSOCKET-PROGRESS] Broadcasting progress data: {progress_data}") + logger.info(f"[WEBSOCKET-PROGRESS] About to broadcast progress message for download {event.download_id}") + await self.connection_manager.broadcast(json.dumps(progress_data)) + logger.info(f"[WEBSOCKET-PROGRESS] Progress message broadcast completed for download {event.download_id}") + + except Exception as e: + logger.warning(f"Failed to send progress update for {event.download_id}: {e}") + + async def on_error(self, event: ProgressEvent): + """Send error update to WebSocket clients""" + logger.info(f"[WEBSOCKET-ERROR] Received error event for download {event.download_id}: {event.message}") + try: + error_data = { + "type": "error", + "id": event.download_id, + "download_id": event.download_id, + "status": "failed", + "message": event.message, + "data": event.data, + "timestamp": event.timestamp.isoformat() + } + + # Update active_downloads - need to find the web interface download ID + # that corresponds to this BDFR download ID + web_download_id = None + for download_id, download_info in active_downloads.items(): + if download_info.get("bdfr_download_id") == event.download_id: + web_download_id = download_id + break + + if web_download_id: + active_downloads[web_download_id]["status"] = "failed" + active_downloads[web_download_id]["error"] = event.message + active_downloads[web_download_id]["end_time"] = datetime.now().isoformat() + # Persist failure phase for status_update broadcasting (e.g., 'rate_limited') + try: + if isinstance(event.data, dict) and event.data.get("phase"): + active_downloads[web_download_id]["phase"] = event.data.get("phase") + except Exception: + pass + logger.info(f"Updated download {web_download_id} to failed status") + + # Unify IDs for frontend and include mapping + error_data["id"] = web_download_id + error_data["download_id"] = web_download_id + error_data["bdfr_download_id"] = event.download_id + error_data["web_download_id"] = web_download_id + error_data["subreddit"] = active_downloads[web_download_id].get("subreddit") + error_data["username"] = active_downloads[web_download_id].get("username") + # Also surface phase at top-level for clients that check data.phase or phase + try: + if isinstance(event.data, dict) and event.data.get("phase"): + error_data["phase"] = event.data.get("phase") + except Exception: + pass + else: + error_data["bdfr_download_id"] = event.download_id + + await self.connection_manager.broadcast(json.dumps(error_data)) + + except Exception as e: + logger.warning(f"Failed to send error update for {event.download_id}: {e}") + logger.warning(f"Error data type: {type(event.data.get('exception'))}") + + async def on_completed(self, event: ProgressEvent): + """Send completion update to WebSocket clients""" + logger.info(f"[WEBSOCKET-COMPLETED] Received completion event for download {event.download_id}: {event.message}") + try: + completed_data = { + "type": "completed", + "id": event.download_id, + "download_id": event.download_id, + "status": "completed", + "message": event.message, + "progress": 100.0, + "data": event.data, + "timestamp": event.timestamp.isoformat() + } + + # Update active_downloads - need to find the web interface download ID + # that corresponds to this BDFR download ID + web_download_id = None + for download_id, download_info in active_downloads.items(): + if download_info.get("bdfr_download_id") == event.download_id: + web_download_id = download_id + break + + if web_download_id: + active_downloads[web_download_id]["status"] = "completed" + active_downloads[web_download_id]["progress"] = 100.0 + active_downloads[web_download_id]["end_time"] = datetime.now().isoformat() + logger.info(f"Updated download {web_download_id} to completed status") + + # Unify IDs and include mapping data + completed_data["id"] = web_download_id + completed_data["download_id"] = web_download_id + completed_data["bdfr_download_id"] = event.download_id + completed_data["web_download_id"] = web_download_id + completed_data["subreddit"] = active_downloads[web_download_id].get("subreddit") + completed_data["username"] = active_downloads[web_download_id].get("username") + else: + completed_data["bdfr_download_id"] = event.download_id + + await self.connection_manager.broadcast(json.dumps(completed_data)) + + except Exception as e: + logger.warning(f"Failed to send completion update for {event.download_id}: {e}") + +# Helper function to create download with BDFR API +async def create_download_with_bdfr_api(download_type: str, name: str, **kwargs): + """Create a download using the new BDFR API layer""" + + # Check if BDFR is available + if not hasattr(bdfr_manager, 'download_subreddit'): + # BDFR not available, create a mock failed download + download_id = f"{download_type}_{name.replace('/', '_').replace(' ', '_')}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + active_downloads[download_id] = { + "id": download_id, + "type": download_type, + "status": "failed", + "progress": 0, + "start_time": datetime.now().isoformat(), + "authenticated": False, + "items_processed": 0, + "items_found": 0, + "current_item": None, + "current_item_type": None, + "phase": "failed", + "error": "BDFR API not available" + } + + # Add type-specific fields + if download_type in ["subreddit", "archive", "clone"]: + active_downloads[download_id]["subreddit"] = name + elif download_type == "user": + active_downloads[download_id]["username"] = name + + return download_id + + # Get global auth token if available + global _bdfr_auth_token + auth_token = _bdfr_auth_token + logger.info(f"[DEBUG] Global auth token available: {auth_token is not None}") + logger.info(f"[DEBUG] Auth token for BDFR: {auth_token[:10]}..." if auth_token else "None") + + # Create unique download ID + download_id = f"{download_type}_{name.replace('/', '_').replace(' ', '_')}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + + # Get auth token if provided + auth_token = None + logger.info(f"[DEBUG] Checking auth_state: {kwargs.get('auth_state')}") + if kwargs.get('auth_state'): + try: + oauth_manager = get_oauth_manager() + logger.info(f"[DEBUG] OAuth manager exists: {oauth_manager is not None}") + auth_token = oauth_manager.get_valid_token(kwargs['auth_state']) + logger.info(f"[DEBUG] Auth token retrieved for state {kwargs['auth_state']}: {auth_token[:10]}..." if auth_token else "None") + except Exception as e: + logger.warning(f"Failed to get auth token: {e}") + + # Initialize download tracking for WebSocket compatibility + active_downloads[download_id] = { + "id": download_id, + "type": download_type, + "status": "queued", + "progress": 0, + "start_time": datetime.now().isoformat(), + "limit": kwargs.get('limit'), + "authenticated": auth_token is not None, + "items_processed": 0, + "items_found": 0, + "current_item": None, + "current_item_type": None, + "phase": "queued" + } + + # Add type-specific fields + if download_type in ["subreddit", "archive", "clone"]: + active_downloads[download_id]["subreddit"] = name + elif download_type == "user": + active_downloads[download_id]["username"] = name + + # Create progress callback + callback = WebSocketProgressCallback(download_id, manager) + + # Start download based on type + if download_type == "subreddit": + bdfr_download_id = bdfr_manager.download_subreddit( + name, + limit=kwargs.get('limit'), + sort=kwargs.get('sort', 'hot'), + time_filter=kwargs.get('time_filter', 'all'), + no_dupes=kwargs.get('no_dupes', False), + progress_callbacks=[callback] + ) + elif download_type == "archive": + # Archive mode - metadata only + bdfr_download_id = bdfr_manager.archive_subreddit( + name, + format_type=kwargs.get('format', 'json'), + limit=kwargs.get('limit'), + progress_callbacks=[callback] + ) + elif download_type == "clone": + # Clone mode - both download and archive + bdfr_download_id = bdfr_manager.clone_subreddit( + name, + limit=kwargs.get('limit'), + format_type=kwargs.get('format', 'json'), + no_dupes=kwargs.get('no_dupes', False), + progress_callbacks=[callback] + ) + elif download_type == "user": + bdfr_download_id = bdfr_manager.download_user( + name, + limit=kwargs.get('limit'), + submitted=kwargs.get('submitted', True), + upvoted=kwargs.get('upvoted', False), + saved=kwargs.get('saved', False), + no_dupes=kwargs.get('no_dupes', False), + progress_callbacks=[callback] + ) + else: + raise ValueError(f"Unsupported download type: {download_type}") + + # Store BDFR download ID for tracking + active_downloads[download_id]["bdfr_download_id"] = bdfr_download_id + + return download_id + +@app.get("/", response_class=HTMLResponse) +async def home(request: Request): + """Serve the main interface""" + return templates.TemplateResponse("index.html", {"request": request}) + +@app.get("/health") +async def health_check(): + """Health check endpoint""" + return {"status": "healthy", "timestamp": datetime.now().isoformat()} + +# OAuth2 Authentication Endpoints +@app.get("/auth/login") +async def oauth_login(redirect_uri: str = None): + """Initiate OAuth2 login flow""" + try: + # Use provided redirect_uri or fall back to configured default + if redirect_uri is None: + redirect_uri = os.getenv("BDFR_REDIRECT_URI", "http://localhost:8000/auth/callback") + + oauth_manager = get_oauth_manager() + auth_data = oauth_manager.get_authorization_url(redirect_uri) + + return { + "authorization_url": auth_data["authorization_url"], + "state": auth_data["state"], + "redirect_uri": redirect_uri, + "message": "Redirect user to the authorization URL" + } + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to initiate OAuth2 login: {str(e)}" + ) + +@app.get("/auth/callback") +async def oauth_callback( + request: Request, + code: str = Query(..., description="Authorization code from Reddit"), + state: str = Query(..., description="State parameter for security"), + error: Optional[str] = Query(None, description="Error from OAuth2 provider") +): + """Handle OAuth2 callback""" + try: + if error: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"OAuth2 error: {error}" + ) + + if not code or not state: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Missing code or state parameter" + ) + + oauth_manager = get_oauth_manager() + # Use the same redirect URI that was used for authorization + redirect_uri = os.getenv("BDFR_REDIRECT_URI", "http://localhost:8000/auth/callback") + token_data = await oauth_manager.exchange_code_for_token(code, state, redirect_uri) + + # Get user info + user_info = await oauth_manager.get_user_info(token_data["access_token"]) + + # Return success page instead of JSON + return templates.TemplateResponse("auth_success.html", {"request": request}) + + except HTTPException: + raise + except Exception as e: + logger.error(f"OAuth2 callback error: {str(e)}") + # Return error page for unexpected errors + return templates.TemplateResponse("auth_error.html", { + "request": request, + "error": "Authentication failed", + "details": str(e) + }) + + except HTTPException: + raise + except Exception as e: + logger.error(f"OAuth2 callback error: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Authentication failed" + ) + +@app.get("/auth/status") +async def auth_status(state: Optional[str] = Query(None)): + """Get current authentication status""" + try: + oauth_manager = get_oauth_manager() + status_data = oauth_manager.get_auth_status(state) + + return { + "authenticated": status_data["authenticated"], + "message": status_data.get("message", "Authenticated" if status_data["authenticated"] else "Not authenticated"), + "expires_at": status_data.get("expires_at"), + "scopes": status_data.get("scopes"), + "username": status_data.get("username") + } + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to get auth status: {str(e)}" + ) + +@app.get("/api/auth/status") +async def api_auth_status(state: Optional[str] = Query(None)): + """Get current authentication status (JSON API)""" + try: + oauth_manager = get_oauth_manager() + + # If no state provided, check if there are any active sessions + if not state and oauth_manager: + # Check if there are any stored access tokens + if oauth_manager.access_tokens: + # Use the first available state for checking + state = next(iter(oauth_manager.access_tokens.keys())) + logger.info(f"No state provided, using first available: {state}") + + status_data = oauth_manager.get_auth_status(state) if oauth_manager else {"authenticated": False, "message": "No OAuth manager"} + + # Debug logging + logger.info(f"Auth status check - State: {state}, Authenticated: {status_data['authenticated']}") + + # Store auth token for BDFR if authenticated + global _bdfr_auth_token + if status_data['authenticated'] and state: + _bdfr_auth_token = oauth_manager.get_valid_token(state) + logger.info(f"Stored auth token for BDFR manager: {_bdfr_auth_token is not None}") + elif not status_data['authenticated']: + _bdfr_auth_token = None + logger.info("Cleared auth token for BDFR manager") + + return { + "authenticated": status_data["authenticated"], + "message": status_data.get("message", "Authenticated" if status_data["authenticated"] else "Not authenticated"), + "expires_at": status_data.get("expires_at", 0), + "scopes": status_data.get("scopes", []), + "username": status_data.get("username"), + "debug_info": { + "state_provided": state is not None, + "state_used": state, + "oauth_manager_exists": oauth_manager is not None, + "available_states": len(oauth_manager.oauth_states) if oauth_manager else 0, + "available_tokens": len(oauth_manager.access_tokens) if oauth_manager else 0, + "bdfr_auth_token_set": _bdfr_auth_token is not None + } + } + except Exception as e: + logger.error(f"Auth status error: {str(e)}") + return { + "authenticated": False, + "message": f"Error checking auth status: {str(e)}", + "error": True, + "debug_info": { + "error_details": str(e) + } + } + +@app.post("/auth/logout") +async def auth_logout(state: str = Form(...)): + """Logout and revoke OAuth2 session""" + try: + oauth_manager = get_oauth_manager() + oauth_manager.revoke_session(state) + + return { + "message": "Successfully logged out", + "state": state + } + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Logout failed: {str(e)}" + ) + +@app.post("/api/download/subreddit") +async def download_subreddit( + subreddit: str = Form(...), + limit: int = Form(10), + sort: str = Form("hot"), + time_filter: str = Form(""), + min_score: str = Form(""), + no_dupes: bool = Form(False), + simple_check: bool = Form(False), + make_hard_links: bool = Form(False), + download_mode: str = Form("download"), + auth_state: str = Form(None) +): + """Download from subreddit using BDFR API""" + + # Map download_mode to appropriate BDFR operation + if download_mode == "archive": + # Use archive mode + download_id = await create_download_with_bdfr_api( + "archive", + subreddit, + limit=limit, + sort=sort, + time_filter=time_filter or "all", + format="json", + simple_check=simple_check, + auth_state=auth_state + ) + message = f"Starting archive for r/{subreddit}" + elif download_mode == "clone": + # Use clone mode (download + archive) + download_id = await create_download_with_bdfr_api( + "clone", + subreddit, + limit=limit, + sort=sort, + time_filter=time_filter or "all", + no_dupes=no_dupes, + simple_check=simple_check, + format="json", + auth_state=auth_state + ) + message = f"Starting clone for r/{subreddit}" + else: + # Default: download mode + download_id = await create_download_with_bdfr_api( + "subreddit", + subreddit, + limit=limit, + sort=sort, + time_filter=time_filter or "all", + no_dupes=no_dupes, + simple_check=simple_check, + auth_state=auth_state + ) + message = f"Starting download for r/{subreddit}" + + return { + "download_id": download_id, + "message": message, + "mode": download_mode, + "estimated_time": "2-3 minutes" + } + +@app.post("/api/download/user") +async def download_user( + username: str = Form(...), + limit: int = Form(10), + submitted: bool = Form(True), + sort: str = Form("hot"), + time_filter: str = Form(""), + no_dupes: bool = Form(False), + simple_check: bool = Form(False), + make_hard_links: bool = Form(False), + download_mode: str = Form("download"), + auth_state: str = Form(None) +): + """Download from user using BDFR API""" + + # Map download_mode to appropriate BDFR operation + if download_mode == "archive": + # Use archive mode + download_id = await create_download_with_bdfr_api( + "archive", + username, + limit=limit, + sort=sort, + time_filter=time_filter or "all", + format="json", + submitted=submitted, + simple_check=simple_check, + auth_state=auth_state + ) + message = f"Starting archive for u/{username}" + elif download_mode == "clone": + # Use clone mode (download + archive) + download_id = await create_download_with_bdfr_api( + "clone", + username, + limit=limit, + sort=sort, + time_filter=time_filter or "all", + no_dupes=no_dupes, + simple_check=simple_check, + format="json", + submitted=submitted, + auth_state=auth_state + ) + message = f"Starting clone for u/{username}" + else: + # Default: download mode + download_id = await create_download_with_bdfr_api( + "user", + username, + limit=limit, + sort=sort, + submitted=submitted, + no_dupes=no_dupes, + simple_check=simple_check, + auth_state=auth_state + ) + message = f"Starting download for u/{username}" + + return { + "download_id": download_id, + "message": message, + "mode": download_mode, + "estimated_time": "1-2 minutes" + } + +@app.get("/api/downloads") +async def get_downloads(): + """Get all active downloads""" + return {"downloads": active_downloads} + +@app.get("/api/downloads/{download_id}") +async def get_download_status(download_id: str): + """Get specific download status""" + if download_id not in active_downloads: + raise HTTPException(status_code=404, detail="Download not found") + return {"download": active_downloads[download_id]} + +@app.delete("/api/downloads/{download_id}") +async def cancel_download(download_id: str): + """Cancel a download""" + if download_id not in active_downloads: + raise HTTPException(status_code=404, detail="Download not found") + + # Cancel in BDFR manager if we have the BDFR download ID + if "bdfr_download_id" in active_downloads[download_id]: + bdfr_manager.cancel_download(active_downloads[download_id]["bdfr_download_id"]) + + active_downloads[download_id]["status"] = "cancelled" + return {"message": "Download cancelled"} + +@app.websocket("/ws/progress") +async def websocket_endpoint(websocket: WebSocket): + """WebSocket endpoint for real-time progress updates""" + logger.info(f"[WEBSOCKET-ENDPOINT] New WebSocket connection attempt from {websocket.client}") + await manager.connect(websocket) + logger.info(f"[WEBSOCKET-ENDPOINT] WebSocket connection established successfully") + logger.info(f"[WEBSOCKET-ENDPOINT] Connection count: {len(manager.active_connections)}") + + # Track last broadcast state to avoid redundant messages + last_broadcast_state = {} + cleanup_counter = 0 + keepalive_counter = 0 + + try: + while True: + # Clean up dead connections periodically (every 10 iterations = 20 seconds) + cleanup_counter += 1 + if cleanup_counter >= 10: + await manager.cleanup_dead_connections() + cleanup_counter = 0 + + # Send keepalive ping every 15 iterations (30 seconds) to prevent timeout + keepalive_counter += 1 + if keepalive_counter >= 15: + try: + await websocket.send_json({"type": "keepalive", "timestamp": datetime.now().isoformat()}) + logger.debug(f"[WEBSOCKET-ENDPOINT] Sent keepalive ping") + except Exception as e: + logger.warning(f"[WEBSOCKET-ENDPOINT] Failed to send keepalive: {e}") + break + keepalive_counter = 0 + + # Check for changes in download states + has_changes = False + for download_id, download in list(active_downloads.items()): + # Only check active downloads + if download.get("status") in ["running", "queued", "failed"]: + # Create a state snapshot for comparison + current_state = { + "status": download.get("status"), + "progress": download.get("progress", 0), + "phase": download.get("phase", "unknown"), + "items_processed": download.get("items_processed", 0), + "items_found": download.get("items_found", 0), + "current_item": download.get("current_item"), + } + + # Compare with last broadcast state + if download_id not in last_broadcast_state or last_broadcast_state[download_id] != current_state: + has_changes = True + last_broadcast_state[download_id] = current_state + + logger.debug(f"[WEBSOCKET-ENDPOINT] State change detected for {download_id}: {download.get('status')}") + status_message = { + "type": "status_update", + "id": download_id, + "download_id": download_id, + "status": download["status"], + "progress": download.get("progress", 0), + "phase": download.get("phase", "unknown"), + "items_processed": download.get("items_processed", 0), + "items_found": download.get("items_found", 0), + "limit": download.get("limit", 0), + "current_item": download.get("current_item"), + "current_item_type": download.get("current_item_type"), + "current_subreddit": download.get("current_subreddit"), + "username": download.get("username"), + "subreddit": download.get("subreddit"), + "message": f"{download.get('subreddit', download.get('username', 'content'))} - {download['status']}" + } + try: + await manager.broadcast(json.dumps(status_message)) + except Exception as e: + logger.warning(f"Error broadcasting status update: {e}") + + if not has_changes: + logger.debug(f"[WEBSOCKET-ENDPOINT] No state changes detected, skipping broadcast") + + await asyncio.sleep(2) # Check every 2 seconds + + except WebSocketDisconnect: + logger.info(f"[WEBSOCKET-ENDPOINT] WebSocket disconnected normally from {websocket.client}") + manager.disconnect(websocket) + except Exception as e: + logger.error(f"[WEBSOCKET-ENDPOINT] WebSocket error from {websocket.client}: {e}") + manager.disconnect(websocket) + +@app.get("/api/bdfr/status") +async def bdfr_status(): + """Get BDFR status and available options""" + return { + "bdfr_available": BDFR_AVAILABLE, + "version": "2.0.0", + "supported_operations": [ + "subreddit_download", + "user_download", + "custom_filter" + ], + "output_formats": ["json", "csv", "xml"] + } + +@app.get("/api/websocket/status") +async def websocket_status(): + """Get WebSocket connection status for debugging""" + return { + "active_connections": len(manager.active_connections), + "connection_failures": manager.connection_failures, + "active_downloads": len(active_downloads), + "websocket_endpoint": "/ws/progress", + "server_host": "0.0.0.0", + "server_port": 8000 + } + +@app.get("/api/oauth/debug") +async def oauth_debug(): + """Debug endpoint to check OAuth state""" + try: + oauth_manager = get_oauth_manager() + return { + "oauth_manager_initialized": oauth_manager is not None, + "oauth_states_count": len(oauth_manager.oauth_states) if oauth_manager else 0, + "access_tokens_count": len(oauth_manager.access_tokens) if oauth_manager else 0, + "refresh_tokens_count": len(oauth_manager.refresh_tokens) if oauth_manager else 0, + "client_id_configured": oauth_manager.client_id != "U-6gk4ZCh3IeNQ" if oauth_manager else False, + "redirect_uri": os.getenv("BDFR_REDIRECT_URI", "http://localhost:8000/auth/callback") + } + except Exception as e: + return { + "error": str(e), + "oauth_manager_initialized": False + } + +if __name__ == "__main__": + import uvicorn + uvicorn.run("app.main:app", host="0.0.0.0", port=8000, reload=True) \ No newline at end of file diff --git a/web_interface/requirements.txt b/web_interface/requirements.txt new file mode 100644 index 0000000..06cfdfe --- /dev/null +++ b/web_interface/requirements.txt @@ -0,0 +1,11 @@ +fastapi>=0.100.0 +uvicorn[standard]>=0.20.0 +websockets>=10.0 +jinja2>=3.1.0 +python-multipart>=0.0.6 +aiofiles>=0.23.0 +python-dotenv>=1.0.0 +pydantic>=2.0.0 +pydantic-settings>=2.0.0 +requests>=2.25.0 +httpx>=0.24.0 \ No newline at end of file diff --git a/web_interface/setup_oauth.py b/web_interface/setup_oauth.py new file mode 100644 index 0000000..0707c39 --- /dev/null +++ b/web_interface/setup_oauth.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +""" +BDFR Web Interface OAuth Setup Helper + +This script helps you set up Reddit OAuth for the BDFR web interface. +Run this script to configure your OAuth credentials and redirect URI. +""" + +import os +import sys +from pathlib import Path + + +def create_env_file(): + """Create .env file from template""" + env_example = Path(__file__).parent / ".env.example" + env_file = Path(__file__).parent / ".env" + + if not env_example.exists(): + print("❌ Error: .env.example not found") + return False + + if env_file.exists(): + print("⚠️ .env file already exists") + response = input("Do you want to overwrite it? (y/N): ").lower().strip() + if response != 'y': + print("Setup cancelled") + return False + + # Copy .env.example to .env + with open(env_example, 'r') as src, open(env_file, 'w') as dst: + dst.write(src.read()) + + print("✅ Created .env file from template") + return True + + +def get_oauth_instructions(): + """Display OAuth setup instructions""" + print("\n" + "="*60) + print("🔐 REDDIT OAUTH SETUP INSTRUCTIONS") + print("="*60) + print() + print("To use the BDFR Web Interface authentication features, you need to:") + print() + print("1. 📱 CREATE OR UPDATE REDDIT OAUTH APP:") + print(" • Go to: https://www.reddit.com/prefs/apps") + print(" • Find your app or click 'Create App'") + print(" • Set the redirect URI to: http://localhost:8000/auth/callback") + print() + print("2. 📝 COPY YOUR CREDENTIALS:") + print(" • After creating/editing the app, copy the client ID and secret") + print(" • These are the values that look like: 7CZHY6AmKweZME5s50SfDGylaPg") + print() + print("3. ✏️ EDIT YOUR CONFIGURATION:") + print(" • Open the .env file that was just created") + print(" • Update BDFR_REDIRECT_URI if using a different port/domain") + print(" • Update BDFR_CLIENT_ID with your OAuth client ID") + print(" • Update BDFR_CLIENT_SECRET with your OAuth client secret") + print(" • OR update bdfr/default_config.cfg with your OAuth credentials") + print() + print("💡 TIP: Use the .env file for web interface configuration") + print(" and bdfr/default_config.cfg for CLI tool configuration") + print() + print("="*60) + print() + + input("Press Enter to open the .env file for editing...") + return True + + +def open_env_file(): + """Open .env file in default editor""" + env_file = Path(__file__).parent / ".env" + + if not env_file.exists(): + print("❌ Error: .env file not found") + return False + + print(f"📝 Opening {env_file} for editing...") + + # Try to open with default editor + editor = os.getenv('EDITOR', 'notepad' if os.name == 'nt' else 'nano') + + try: + if os.name == 'nt': # Windows + os.startfile(env_file) + else: # Unix-like + os.system(f"{editor} {env_file}") + return True + except Exception as e: + print(f"❌ Error opening editor: {e}") + print(f"📍 Please manually edit the file: {env_file}") + return False + + +def main(): + """Main setup function""" + print("🚀 BDFR Web Interface OAuth Setup") + print("=" * 40) + + # Create .env file + if not create_env_file(): + return 1 + + # Show instructions + if not get_oauth_instructions(): + return 1 + + # Open .env file for editing + if not open_env_file(): + print("📝 Please manually edit the .env file with your OAuth settings") + print("📍 File location:", Path(__file__).parent / ".env") + + print("\n✅ OAuth setup initiated!") + print("📖 Check STARTUP.md for detailed setup instructions") + print("🚀 Run 'python start.py' to start the web interface after configuration") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/web_interface/start.bat b/web_interface/start.bat new file mode 100644 index 0000000..0e806e6 --- /dev/null +++ b/web_interface/start.bat @@ -0,0 +1,105 @@ +@echo off +REM BDFR Web Interface Startup Script for Windows +REM This script provides an easy way to start the BDFR web interface on Windows + +setlocal enabledelayedexpansion + +REM Colors for output (Windows 10+) +set "RED=[91m" +set "GREEN=[92m" +set "YELLOW=[93m" +set "BLUE=[94m" +set "NC=[0m" + +REM Function to print colored output (simplified for Windows) +echo 🌟 BDFR Web Interface Startup +echo ================================================== + +REM Check if we're in the right directory +if not exist "requirements.txt" ( + echo ❌ Error: Please run this script from the web_interface directory + echo Usage: start.bat + pause + exit /b 1 +) + +if not exist "app" ( + echo ❌ Error: app directory not found + echo Please make sure you're in the web_interface directory + pause + exit /b 1 +) + +echo ℹ️ Checking Python version... + +REM Check Python version +python --version > temp_python_version.txt 2>&1 +set /p PYTHON_VERSION=nul +if errorlevel 1 ( + echo ❌ Error: Python 3 is required + echo Current version: %PYTHON_VERSION% + pause + exit /b 1 +) + +REM Install dependencies +echo ℹ️ Checking and installing dependencies... +if exist requirements.txt ( + echo ℹ️ Installing Python dependencies... + python -m pip install -r requirements.txt + if !errorlevel! neq 0 ( + echo ❌ Failed to install dependencies + pause + exit /b 1 + ) + echo ✅ Dependencies installed successfully +) else ( + echo ❌ requirements.txt not found + pause + exit /b 1 +) + +REM Check if BDFR module is available +echo ℹ️ Checking BDFR module availability... +python -c "import sys; sys.path.insert(0, '../bdfr'); import bdfr.api; print('BDFR API imported successfully')" >nul 2>&1 +if !errorlevel! neq 0 ( + echo ⚠️ Warning: BDFR module not found in Python path + echo Make sure the parent directory is in your Python path + echo Or run this script from the project root directory + echo Attempting to install BDFR... + cd .. + python -m pip install -e . + cd web_interface + if !errorlevel! neq 0 ( + echo ❌ Failed to install BDFR + pause + exit /b 1 + ) + echo ✅ BDFR installed successfully +) else ( + echo ✅ BDFR module found +) + +REM Start the server +echo. +echo ================================================== +echo ℹ️ Starting BDFR Web Interface... +echo ℹ️ Server will be available at: http://localhost:8000 +echo ℹ️ API documentation at: http://localhost:8000/docs +echo ℹ️ Press Ctrl+C to stop the server +echo ================================================== + +REM Start uvicorn server +python -m uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload + +REM This code runs when the server is stopped +echo. +echo ℹ️ BDFR Web Interface stopped + +pause \ No newline at end of file diff --git a/web_interface/start.py b/web_interface/start.py new file mode 100644 index 0000000..3d9a806 --- /dev/null +++ b/web_interface/start.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +""" +BDFR Web Interface Startup Script + +This script provides an easy way to start the BDFR web interface with +proper dependency management and error handling. +""" + +import os +import sys +import subprocess +import importlib.util +from pathlib import Path + + +def check_python_version(): + """Check if Python version is compatible (3.8+)""" + if sys.version_info < (3, 8): + print("ERROR: Python 3.8 or higher is required") + print(f"Current version: {sys.version}") + sys.exit(1) + + +def install_dependencies(): + """Install required dependencies if missing""" + requirements_path = Path(__file__).parent / "requirements.txt" + + if not requirements_path.exists(): + print("❌ Error: requirements.txt not found") + sys.exit(1) + + print("Checking and installing dependencies...") + + try: + # Try to import required modules first + required_modules = [ + 'fastapi', + 'uvicorn', + 'websockets', + 'jinja2' + ] + + missing_modules = [] + for module in required_modules: + if not importlib.util.find_spec(module): + missing_modules.append(module) + + if missing_modules: + print(f"Installing missing modules: {', '.join(missing_modules)}") + subprocess.check_call([ + sys.executable, '-m', 'pip', 'install', '-r', str(requirements_path) + ]) + else: + print("All dependencies are already installed") + + except subprocess.CalledProcessError as e: + print(f"❌ Error installing dependencies: {e}") + sys.exit(1) + except Exception as e: + print(f"❌ Error checking dependencies: {e}") + sys.exit(1) + + +def check_bdfr_module(): + """Check if BDFR module is available""" + try: + importlib.util.find_spec('bdfr') + print("BDFR module found") + except ImportError: + print("⚠️ Warning: BDFR module not found in Python path") + print("Make sure the parent directory is in your Python path or run from project root") + + +def start_server(): + """Start the FastAPI server""" + print("Starting BDFR Web Interface...") + print("Server will be available at: http://localhost:8000") + print("API documentation at: http://localhost:8000/docs") + print("Press Ctrl+C to stop the server") + print("-" * 50) + + try: + # Start uvicorn server + subprocess.call([ + sys.executable, '-m', 'uvicorn', + 'app.main:app', + '--host', '0.0.0.0', + '--port', '8000', + '--reload' + ]) + except KeyboardInterrupt: + print("\n🛑 Server stopped by user") + except Exception as e: + print(f"❌ Error starting server: {e}") + sys.exit(1) + + +def main(): + """Main startup function""" + print("BDFR Web Interface Startup") + print("=" * 40) + + # Change to web_interface directory + web_interface_dir = Path(__file__).parent + os.chdir(web_interface_dir) + + # Pre-flight checks + check_python_version() + install_dependencies() + check_bdfr_module() + + print("\n" + "=" * 40) + + # Start the server + start_server() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/web_interface/start.sh b/web_interface/start.sh new file mode 100644 index 0000000..a7d10dc --- /dev/null +++ b/web_interface/start.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# BDFR Web Interface Startup Script +# Compatible with Linux, macOS, and other Unix-like systems + +set -e # Exit on any error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Function to print colored output +print_info() { + echo -e "${BLUE}ℹ️ $1${NC}" +} + +print_success() { + echo -e "${GREEN}✅ $1${NC}" +} + +print_warning() { + echo -e "${YELLOW}⚠️ $1${NC}" +} + +print_error() { + echo -e "${RED}❌ $1${NC}" +} + +# Check if we're in the right directory +if [[ ! -f "requirements.txt" ]] || [[ ! -d "app" ]]; then + print_error "Error: Please run this script from the web_interface directory" + echo "Usage: ./start.sh" + exit 1 +fi + +print_info "BDFR Web Interface Startup" +echo "==================================================" + +# Check Python version +print_info "Checking Python version..." +PYTHON_VERSION=$(python3 --version 2>&1 | awk '{print $2}') +print_success "Python version: $PYTHON_VERSION" + +# Check if Python 3.8+ is available +PYTHON_MAJOR=$(echo $PYTHON_VERSION | cut -d. -f1) +PYTHON_MINOR=$(echo $PYTHON_VERSION | cut -d. -f2) + +if [[ $PYTHON_MAJOR -lt 3 ]] || [[ $PYTHON_MAJOR -eq 3 && $PYTHON_MINOR -lt 8 ]]; then + print_error "Python 3.8 or higher is required" + print_error "Current version: $PYTHON_VERSION" + exit 1 +fi + +# Install dependencies +print_info "Checking and installing dependencies..." +if [[ -f "requirements.txt" ]]; then + # Check if pip is available + if ! command -v pip3 &> /dev/null; then + print_error "pip3 is not installed. Please install Python 3 and pip first." + exit 1 + fi + + # Install/update requirements + print_info "Installing Python dependencies..." + pip3 install -r requirements.txt + + if [[ $? -eq 0 ]]; then + print_success "Dependencies installed successfully" + else + print_error "Failed to install dependencies" + exit 1 + fi +else + print_error "requirements.txt not found" + exit 1 +fi + +# Check if BDFR module is available +print_info "Checking BDFR module availability..." +if python3 -c "import bdfr" 2>/dev/null; then + print_success "BDFR module found" +else + print_warning "BDFR module not found in Python path" + print_warning "Make sure the parent directory is in your Python path" + print_warning "Or run this script from the project root directory" +fi + +# Start the server +echo "" +echo "==================================================" +print_info "Starting BDFR Web Interface..." +print_info "Server will be available at: http://localhost:8000" +print_info "API documentation at: http://localhost:8000/docs" +print_info "Press Ctrl+C to stop the server" +echo "==================================================" + +# Start uvicorn server with proper error handling +python3 -m uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload + +# This code runs when the server is stopped +echo "" +print_info "BDFR Web Interface stopped" \ No newline at end of file diff --git a/web_interface/static/css/style.css b/web_interface/static/css/style.css new file mode 100644 index 0000000..efeb1e1 --- /dev/null +++ b/web_interface/static/css/style.css @@ -0,0 +1,817 @@ +/* Reset and base styles */ +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif; + line-height: 1.6; + color: #333; + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + min-height: 100vh; +} + +.container { + max-width: 1200px; + margin: 0 auto; + padding: 20px; +} + +/* Header */ +header { + text-align: center; + margin-bottom: 40px; + color: white; + position: relative; +} + +header h1 { + font-size: 3rem; + font-weight: 700; + margin-bottom: 10px; + text-shadow: 0 2px 4px rgba(0,0,0,0.3); +} + +.subtitle { + font-size: 1.2rem; + opacity: 0.9; + font-weight: 300; +} + +/* Authentication section */ +.auth-section { + margin-top: 20px; + padding: 15px; + background: rgba(255, 255, 255, 0.1); + border-radius: 10px; + backdrop-filter: blur(10px); + border: 1px solid rgba(255, 255, 255, 0.2); +} + +.auth-status { + display: flex; + justify-content: space-between; + align-items: center; + flex-wrap: wrap; + gap: 15px; +} + +.auth-info { + display: flex; + align-items: center; + gap: 10px; + flex-wrap: wrap; +} + +.auth-label { + font-weight: 600; + color: rgba(255, 255, 255, 0.9); +} + +.auth-user { + font-weight: 500; + color: white; + background: rgba(255, 255, 255, 0.2); + padding: 4px 8px; + border-radius: 4px; + font-size: 0.9rem; +} + +.auth-status-indicator { + font-weight: 600; + padding: 4px 8px; + border-radius: 4px; + font-size: 0.85rem; +} + +.auth-status-indicator.connected { + background: rgba(40, 167, 69, 0.2); + color: #28a745; + border: 1px solid rgba(40, 167, 69, 0.3); +} + +.auth-status-indicator.disconnected { + background: rgba(220, 53, 69, 0.2); + color: #dc3545; + border: 1px solid rgba(220, 53, 69, 0.3); +} + +.auth-actions { + display: flex; + gap: 10px; +} + +/* Main content */ +main { + background: white; + border-radius: 15px; + padding: 30px; + box-shadow: 0 10px 30px rgba(0,0,0,0.2); + margin-bottom: 30px; +} + +/* Download section */ +.download-section { + margin-bottom: 40px; +} + +.form-container { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(400px, 1fr)); + gap: 30px; + margin-bottom: 30px; +} + +.form-container-unified { + max-width: 900px; + margin: 0 auto; +} + +.form-card { + background: #f8f9fa; + padding: 25px; + border-radius: 10px; + border-left: 4px solid #667eea; + box-shadow: 0 2px 10px rgba(0,0,0,0.1); +} + +.form-card-unified { + background: #f8f9fa; + padding: 30px; + border-radius: 10px; + border-left: 4px solid #667eea; + box-shadow: 0 2px 10px rgba(0,0,0,0.1); +} + +.form-card h2, +.form-card-unified h2 { + color: #2c3e50; + margin-bottom: 20px; + font-size: 1.5rem; +} + +.form-card:nth-child(2) { + border-left-color: #764ba2; +} + +/* Form styles */ +.download-form { + display: flex; + flex-direction: column; + gap: 15px; +} + +.form-group { + display: flex; + flex-direction: column; +} + +.form-group label { + margin-bottom: 5px; + font-weight: 600; + color: #555; +} + +.form-group input, +.form-group select { + padding: 12px; + border: 2px solid #e1e8ed; + border-radius: 6px; + font-size: 1rem; + transition: border-color 0.3s ease; +} + +.form-group input:focus, +.form-group select:focus { + outline: none; + border-color: #667eea; + box-shadow: 0 0 0 3px rgba(102, 126, 234, 0.1); +} + +/* Form layout enhancements */ +.form-row { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 15px; +} + +.form-section { + margin-top: 25px; + padding-top: 20px; + border-top: 1px solid #e9ecef; +} + +.form-section h4 { + color: #2c3e50; + font-size: 1.1rem; + margin-bottom: 15px; + display: flex; + align-items: center; + gap: 8px; +} + +.form-help { + font-size: 0.85rem; + color: #666; + margin-top: 4px; + line-height: 1.4; +} + +/* Checkbox styles */ +.checkbox-group { + display: flex; + flex-direction: column; + gap: 12px; +} + +.checkbox-label { + display: flex; + align-items: center; + cursor: pointer; + font-weight: 500; + padding: 8px; + border-radius: 6px; + transition: background-color 0.2s ease; +} + +.checkbox-label:hover { + background-color: #f0f2f5; +} + +.checkbox-label input[type="checkbox"] { + display: none; +} + +.checkmark { + width: 20px; + height: 20px; + border: 2px solid #ddd; + border-radius: 4px; + margin-right: 10px; + position: relative; + transition: all 0.2s ease; +} + +.checkbox-label input[type="checkbox"]:checked + .checkmark { + background-color: #667eea; + border-color: #667eea; +} + +.checkbox-label input[type="checkbox"]:checked + .checkmark::after { + content: '✓'; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + color: white; + font-size: 14px; + font-weight: bold; +} + +/* Radio button styles */ +.radio-group { + display: flex; + flex-direction: column; + gap: 10px; +} + +.radio-label { + display: flex; + align-items: center; + cursor: pointer; + font-weight: 500; + padding: 8px; + border-radius: 6px; + transition: background-color 0.2s ease; +} + +.radio-label:hover { + background-color: #f0f2f5; +} + +.radio-label input[type="radio"] { + display: none; +} + +.radio-custom { + width: 20px; + height: 20px; + border: 2px solid #ddd; + border-radius: 50%; + margin-right: 10px; + position: relative; + transition: all 0.2s ease; +} + +.radio-label input[type="radio"]:checked + .radio-custom { + border-color: #667eea; +} + +.radio-label input[type="radio"]:checked + .radio-custom::after { + content: ''; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + width: 10px; + height: 10px; + background-color: #667eea; + border-radius: 50%; +} + +/* Mode Selection Styles */ +.mode-section { + background: white; + padding: 20px; + border-radius: 8px; + margin-bottom: 25px; + border: 2px solid #e1e8ed; +} + +.mode-radio-group { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); + gap: 15px; +} + +.mode-option { + background: white; + border: 2px solid #e1e8ed; + border-radius: 8px; + padding: 15px 12px; + transition: all 0.3s ease; + position: relative; +} + +.mode-option:hover { + background-color: #f8f9fa; + border-color: #667eea; + transform: translateY(-2px); + box-shadow: 0 4px 12px rgba(102, 126, 234, 0.15); +} + +.mode-option input[type="radio"]:checked + .radio-custom + .mode-label { + color: #667eea; +} + +.mode-option input[type="radio"]:checked { + & ~ * { + border-color: #667eea; + } +} + +.mode-option input[type="radio"]:checked + .radio-custom { + border-color: #667eea; + background-color: rgba(102, 126, 234, 0.1); +} + +.mode-label { + display: flex; + flex-direction: column; + gap: 4px; + margin-left: 8px; +} + +.mode-label strong { + font-size: 1rem; + color: #2c3e50; +} + +.mode-label small { + font-size: 0.85rem; + color: #666; + font-weight: normal; +} + +/* Tooltip Styles */ +[data-tooltip] { + position: relative; + cursor: help; +} + +[data-tooltip]::before { + content: attr(data-tooltip); + position: absolute; + bottom: calc(100% + 10px); + left: 50%; + transform: translateX(-50%) scale(0.95); + padding: 10px 15px; + background: #2c3e50; + color: white; + border-radius: 6px; + font-size: 0.85rem; + font-weight: 400; + line-height: 1.4; + white-space: normal; + width: max-content; + max-width: 280px; + opacity: 0; + pointer-events: none; + transition: all 0.2s ease; + z-index: 1000; + box-shadow: 0 4px 12px rgba(0,0,0,0.15); +} + +[data-tooltip]::after { + content: ''; + position: absolute; + bottom: calc(100% + 4px); + left: 50%; + transform: translateX(-50%) scale(0.95); + border: 6px solid transparent; + border-top-color: #2c3e50; + opacity: 0; + pointer-events: none; + transition: all 0.2s ease; + z-index: 1000; +} + +[data-tooltip]:hover::before, +[data-tooltip]:hover::after { + opacity: 1; + transform: translateX(-50%) scale(1); +} + +/* Button variations */ +.btn-small { + padding: 8px 16px; + font-size: 0.9rem; +} + +.btn-outline { + background: transparent; + border: 2px solid #6c757d; + color: #6c757d; +} + +.btn-outline:hover { + background: #6c757d; + color: white; +} + +/* Empty state styling */ +.empty-state { + text-align: center; + padding: 40px 20px; + color: #666; +} + +.empty-icon { + font-size: 3rem; + margin-bottom: 15px; + opacity: 0.5; +} + +.empty-state p { + margin-bottom: 8px; +} + +/* Downloads header */ +.downloads-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 15px; +} + +/* Buttons */ +.btn { + padding: 12px 24px; + border: none; + border-radius: 6px; + font-size: 1rem; + font-weight: 600; + cursor: pointer; + transition: all 0.3s ease; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.btn-primary { + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + color: white; +} + +.btn-primary:hover { + transform: translateY(-2px); + box-shadow: 0 5px 15px rgba(102, 126, 234, 0.4); +} + +.btn-secondary { + background: linear-gradient(135deg, #764ba2 0%, #667eea 100%); + color: white; +} + +.btn-secondary:hover { + transform: translateY(-2px); + box-shadow: 0 5px 15px rgba(118, 75, 162, 0.4); +} + +/* Progress section */ +.progress-section { + margin-bottom: 40px; +} + +.progress-section h2 { + color: #2c3e50; + margin-bottom: 20px; + font-size: 1.8rem; +} + +.progress-container { + background: #f8f9fa; + border-radius: 10px; + padding: 20px; + min-height: 100px; + display: flex; + align-items: center; + justify-content: center; +} + +.no-downloads { + text-align: center; + color: #666; +} + +.no-downloads p { + margin-bottom: 10px; +} + +/* Downloads list */ +.downloads-list { + background: #f8f9fa; + border-radius: 10px; + padding: 20px; +} + +.downloads-list h3 { + margin-bottom: 15px; + color: #2c3e50; +} + +.downloads-items { + display: flex; + flex-direction: column; + gap: 15px; +} + +/* Progress card */ +.progress-card { + background: white; + padding: 20px; + border-radius: 8px; + border-left: 4px solid #28a745; + box-shadow: 0 2px 8px rgba(0,0,0,0.1); +} + +.progress-card.downloading { + border-left-color: #ffc107; + animation: pulse 2s infinite; +} + +.progress-card.running { + border-left-color: #ffc107; + animation: pulse 2s infinite; +} + +.progress-card.completed { + border-left-color: #28a745; +} + +.progress-card.failed { + border-left-color: #dc3545; +} + +.progress-card.error { + border-left-color: #dc3545; +} + +.progress-header { + display: flex; + justify-content: between; + align-items: center; + margin-bottom: 15px; +} + +.progress-info h4 { + color: #2c3e50; + margin-bottom: 5px; +} + +.progress-meta { + font-size: 0.9rem; + color: #666; +} + +.progress-status { + font-weight: 600; + padding: 4px 8px; + border-radius: 4px; + font-size: 0.8rem; +} + +.status-starting { + background: #e7f3ff; + color: #0066cc; +} + +.status-downloading { + background: #fff3cd; + color: #856404; +} + +.status-completed { + background: #d4edda; + color: #155724; +} + +.status-error { + background: #f8d7da; + color: #721c24; +} + +/* Progress bar */ +.progress-bar-container { + margin-bottom: 10px; +} + +.progress-bar { + width: 100%; + height: 8px; + background: #e9ecef; + border-radius: 4px; + overflow: hidden; +} + +.progress-fill { + height: 100%; + background: linear-gradient(90deg, #667eea 0%, #764ba2 100%); + border-radius: 4px; + transition: width 0.3s ease; +} + +.progress-text { + font-size: 0.9rem; + color: #666; + margin-bottom: 8px; +} + +.progress-details { + font-size: 0.8rem; + color: #888; + margin-top: 8px; +} + +.progress-details div { + margin-bottom: 2px; +} + +.progress-phase { + font-size: 0.85rem; + color: #555; + font-weight: 500; + margin-top: 4px; +} + +.current-item { + font-size: 0.85rem; + color: #007bff; + font-weight: 500; + margin-top: 4px; + padding: 4px 8px; + background-color: rgba(0, 123, 255, 0.1); + border-radius: 4px; + border-left: 3px solid #007bff; +} + +/* Progress controls buttons */ +.progress-controls { + display: flex; + gap: 8px; + align-items: center; +} + +.btn-retry { + padding: 6px 12px; + background: #ffc107; + color: #000; + border: none; + border-radius: 4px; + font-size: 0.85rem; + font-weight: 600; + cursor: pointer; + transition: all 0.2s ease; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.btn-retry:hover { + background: #ffca2c; + transform: translateY(-1px); + box-shadow: 0 2px 8px rgba(255, 193, 7, 0.4); +} + +.btn-retry:active { + transform: translateY(0); +} + +/* Status section */ +.status-section { + margin-bottom: 30px; +} + +.status-card { + background: #f8f9fa; + padding: 20px; + border-radius: 10px; + border-left: 4px solid #17a2b8; +} + +.status-card h3 { + color: #2c3e50; + margin-bottom: 15px; +} + +.status-item { + display: flex; + justify-content: space-between; + margin-bottom: 10px; + padding: 8px 0; + border-bottom: 1px solid #e9ecef; +} + +.status-item:last-child { + border-bottom: none; +} + +.status-label { + font-weight: 600; + color: #555; +} + +.status-value { + font-weight: 500; +} + +.status-online { + color: #28a745; +} + +.status-offline { + color: #dc3545; +} + +/* Footer */ +footer { + text-align: center; + color: white; + opacity: 0.8; +} + +/* Animations */ +@keyframes pulse { + 0% { box-shadow: 0 2px 8px rgba(0,0,0,0.1); } + 50% { box-shadow: 0 4px 16px rgba(255, 193, 7, 0.3); } + 100% { box-shadow: 0 2px 8px rgba(0,0,0,0.1); } +} + +/* Responsive design */ +@media (max-width: 768px) { + .container { + padding: 15px; + } + + header h1 { + font-size: 2.5rem; + } + + .form-container { + grid-template-columns: 1fr; + } + + .form-card { + margin-bottom: 20px; + } + + .status-item { + flex-direction: column; + align-items: flex-start; + gap: 5px; + } +} + +@media (max-width: 480px) { + header h1 { + font-size: 2rem; + } + + main { + padding: 20px; + } + + .form-card { + padding: 20px; + } +} \ No newline at end of file diff --git a/web_interface/static/js/app.js b/web_interface/static/js/app.js new file mode 100644 index 0000000..850105a --- /dev/null +++ b/web_interface/static/js/app.js @@ -0,0 +1,1030 @@ +// WebSocket and application logic for BDFR Web Interface + +class BDFRApp { + constructor() { + this.ws = null; + this.reconnectAttempts = 0; + this.maxReconnectAttempts = 5; + this.reconnectDelay = 1000; + this.downloads = new Map(); + this.authState = null; + this.authenticated = false; + + this.initializeElements(); + this.bindEvents(); + this.connectWebSocket(); + this.updateStatus(); + this.startStatusPolling(); + + // Check for stored auth state first + this.authState = this.getStoredAuthState(); + this.checkAuthentication(); + } + + initializeElements() { + // Forms + this.unifiedForm = document.getElementById('unifiedForm'); + this.subredditForm = document.getElementById('subredditForm'); + this.userForm = document.getElementById('userForm'); + + // Progress containers + this.progressContainer = document.getElementById('progressContainer'); + this.downloadsList = document.getElementById('downloadsList'); + this.downloadsItems = document.getElementById('downloadsItems'); + + // Status elements + this.wsStatus = document.getElementById('wsStatus'); + this.bdfrStatus = document.getElementById('bdfrStatus'); + + // Authentication elements + this.authSection = document.getElementById('authSection'); + this.authUser = document.getElementById('authUser'); + this.authStatus = document.getElementById('authStatus'); + this.loginBtn = document.getElementById('loginBtn'); + this.logoutBtn = document.getElementById('logoutBtn'); + this.authStateInput = document.getElementById('authState'); + this.userAuthStateInput = document.getElementById('userAuthState'); + } + + bindEvents() { + // Form submissions + if (this.unifiedForm) { + this.unifiedForm.addEventListener('submit', (e) => this.handleUnifiedSubmit(e)); + + // Source type toggle + const sourceTypeRadios = this.unifiedForm.querySelectorAll('input[name="source_type"]'); + sourceTypeRadios.forEach(radio => { + radio.addEventListener('change', (e) => this.updateSourceTypeUI(e.target.value)); + }); + } + + if (this.subredditForm) { + this.subredditForm.addEventListener('submit', (e) => this.handleSubredditSubmit(e)); + } + if (this.userForm) { + this.userForm.addEventListener('submit', (e) => this.handleUserSubmit(e)); + } + + // Authentication events + if (this.loginBtn) { + this.loginBtn.addEventListener('click', () => this.handleLogin()); + } + if (this.logoutBtn) { + this.logoutBtn.addEventListener('click', () => this.handleLogout()); + } + // Real-time input validation + ['subreddit', 'username', 'sourceName'].forEach(id => { + const input = document.getElementById(id); + if (input) { + input.addEventListener('input', (e) => this.validateInput(e.target)); + } + }); + } + + async connectWebSocket() { + try { + // Force HTTP for development - change this for production + const protocol = 'ws:'; + const host = window.location.hostname || 'localhost'; + const port = window.location.port || '8000'; + const wsUrl = `${protocol}//${host}:${port}/ws/progress`; + + console.log('[FRONTEND-WS] Attempting to connect to WebSocket:', wsUrl); + console.log('[FRONTEND-WS] Window location:', window.location.href); + console.log('[FRONTEND-WS] Host:', window.location.host); + console.log('[FRONTEND-WS] Hostname:', window.location.hostname); + console.log('[FRONTEND-WS] Port:', window.location.port); + + this.ws = new WebSocket(wsUrl); + + this.ws.onopen = () => { + console.log('[FRONTEND-WS] WebSocket connected successfully'); + this.reconnectAttempts = 0; + this.updateWSStatus('Connected'); + }; + + this.ws.onmessage = (event) => { + console.log('[FRONTEND-WS] Received message:', event.data); + const data = JSON.parse(event.data); + this.handleWebSocketMessage(data); + }; + + this.ws.onclose = (event) => { + console.log('[FRONTEND-WS] WebSocket disconnected:', event.code, event.reason); + this.updateWSStatus('Disconnected'); + this.scheduleReconnect(); + }; + + this.ws.onerror = (error) => { + console.error('[FRONTEND-WS] WebSocket error:', error); + this.updateWSStatus('Error'); + }; + + } catch (error) { + console.error('Failed to connect WebSocket:', error); + this.scheduleReconnect(); + } + } + + scheduleReconnect() { + if (this.reconnectAttempts < this.maxReconnectAttempts) { + this.reconnectAttempts++; + const delay = this.reconnectDelay * Math.pow(2, this.reconnectAttempts - 1); + + console.log(`[FRONTEND-WS] Scheduling reconnect attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts} in ${delay}ms`); + + setTimeout(() => { + console.log(`[FRONTEND-WS] Attempting to reconnect (${this.reconnectAttempts}/${this.maxReconnectAttempts})`); + this.connectWebSocket(); + }, delay); + } else { + console.error('[FRONTEND-WS] Failed to connect after maximum attempts'); + this.updateWSStatus('Failed to connect'); + } + } + + handleWebSocketMessage(data) { + console.log('WebSocket message received:', data); + + // Ensure data has the required fields + if (!data.download_id && data.id) { + data.download_id = data.id; + } + + switch (data.type) { + case 'progress': + case 'status_update': + this.updateProgress(data); + break; + case 'status': + this.showNotification(`Download ${data.download_id}: ${data.message}`, 'info'); + break; + case 'completed': + this.showSuccess(`Download ${data.download_id} completed!`); + this.updateProgress(data); // Also update progress for completion + break; + case 'error': + this.showError(`Download ${data.download_id} failed: ${data.message}`); + this.updateProgress(data); // Also update progress for errors + break; + default: + console.log('Unknown message type:', data.type, data); + } + } + + async handleSubredditSubmit(e) { + e.preventDefault(); + + const formData = new FormData(e.target); + const subreddit = formData.get('subreddit').trim(); + + if (!this.validateSubreddit(subreddit)) { + this.showError('Please enter a valid subreddit name'); + return; + } + + // Get no_dupes checkbox value and add it to formData + const noDupes = document.getElementById('noDupes').checked; + formData.set('no_dupes', noDupes ? 'true' : 'false'); + + // Show additional options for confirmation + const limit = formData.get('limit'); + const sort = formData.get('sort'); + + if (confirm(`Start download for r/${subreddit}?\n\nOptions:\n- Limit: ${limit}\n- Sort: ${sort}\n- No duplicates: ${noDupes ? 'Yes' : 'No'}`)) { + try { + this.showLoading(e.target.querySelector('button')); + + const response = await fetch('/api/download/subreddit', { + method: 'POST', + body: formData + }); + + const result = await response.json(); + + if (response.ok) { + this.showSuccess(`Subreddit download started! ID: ${result.download_id}`); + e.target.reset(); + } else { + this.showError(result.detail || 'Failed to start download'); + } + + } catch (error) { + console.error('Error:', error); + this.showError('Network error occurred'); + } finally { + this.hideLoading(e.target.querySelector('button')); + } + } + } + + async handleUserSubmit(e) { + e.preventDefault(); + + const formData = new FormData(e.target); + const username = formData.get('username').trim(); + + if (!this.validateUsername(username)) { + this.showError('Please enter a valid username'); + return; + } + + // Get no_dupes checkbox value and add it to formData + const noDupes = document.getElementById('userNoDupes').checked; + formData.set('no_dupes', noDupes ? 'true' : 'false'); + + // Get content type for confirmation + const contentType = formData.get('content_type'); + const contentTypeLabel = contentType === 'submitted' ? 'submitted posts' : + contentType === 'upvoted' ? 'upvoted posts' : 'saved posts'; + + if (confirm(`Start download for u/${username}?\n\nContent: ${contentTypeLabel}\nLimit: ${formData.get('limit')}`)) { + try { + this.showLoading(e.target.querySelector('button')); + + const response = await fetch('/api/download/user', { + method: 'POST', + body: formData + }); + + const result = await response.json(); + + if (response.ok) { + this.showSuccess(`User download started! ID: ${result.download_id}`); + e.target.reset(); + } else { + this.showError(result.detail || 'Failed to start download'); + } + + } catch (error) { + console.error('Error:', error); + this.showError('Network error occurred'); + } finally { + this.hideLoading(e.target.querySelector('button')); + } + } + } + + updateSourceTypeUI(sourceType) { + const sourceNameLabel = document.getElementById('sourceNameLabel'); + const sourceNameInput = document.getElementById('sourceName'); + const sourceNameHelp = document.getElementById('sourceNameHelp'); + + if (sourceType === 'subreddit') { + sourceNameLabel.textContent = 'Subreddit Name:'; + sourceNameInput.placeholder = 'e.g., python, machinelearning'; + sourceNameHelp.textContent = "Enter subreddit name without 'r/'"; + } else { + sourceNameLabel.textContent = 'Username:'; + sourceNameInput.placeholder = 'e.g., spez, your_username'; + sourceNameHelp.textContent = "Enter Reddit username without 'u/'"; + } + } + + async handleUnifiedSubmit(e) { + e.preventDefault(); + + const formData = new FormData(e.target); + const downloadMode = formData.get('download_mode'); + const sourceType = formData.get('source_type'); + const sourceName = formData.get('source_name').trim(); + const limit = formData.get('limit'); + const sort = formData.get('sort'); + const noDupes = document.getElementById('noDupes').checked; + const simpleCheck = document.getElementById('simpleCheck').checked; + + // Validate source name + if (sourceType === 'subreddit' && !this.validateSubreddit(sourceName)) { + this.showError('Please enter a valid subreddit name'); + return; + } + if (sourceType === 'user' && !this.validateUsername(sourceName)) { + this.showError('Please enter a valid username'); + return; + } + + // Build confirmation message + const modeLabels = { + 'download': 'Download (media files)', + 'archive': 'Archive (metadata only)', + 'clone': 'Clone (media + metadata)' + }; + const sourceLabel = sourceType === 'subreddit' ? `r/${sourceName}` : `u/${sourceName}`; + + if (confirm(`Start ${modeLabels[downloadMode]} for ${sourceLabel}?\n\nOptions:\n- Limit: ${limit}\n- Sort: ${sort}\n- No duplicates: ${noDupes ? 'Yes' : 'No'}\n- Simple check: ${simpleCheck ? 'Yes' : 'No'}`)) { + try { + this.showLoading(e.target.querySelector('button')); + + // Prepare form data for backend + const backendFormData = new FormData(); + if (sourceType === 'subreddit') { + backendFormData.append('subreddit', sourceName); + } else { + backendFormData.append('username', sourceName); + } + backendFormData.append('limit', limit); + backendFormData.append('sort', sort); + backendFormData.append('time_filter', formData.get('time_filter') || ''); + backendFormData.append('min_score', formData.get('min_score') || ''); + backendFormData.append('no_dupes', noDupes ? 'true' : 'false'); + backendFormData.append('simple_check', simpleCheck ? 'true' : 'false'); + backendFormData.append('make_hard_links', formData.get('make_hard_links') ? 'true' : 'false'); + backendFormData.append('download_mode', downloadMode); + if (this.authState) { + backendFormData.append('auth_state', this.authState); + } + + // Determine endpoint based on source type + const endpoint = sourceType === 'subreddit' + ? '/api/download/subreddit' + : '/api/download/user'; + + // For user downloads, always set submitted=true (default) + if (sourceType === 'user') { + backendFormData.append('submitted', 'true'); + } + + const response = await fetch(endpoint, { + method: 'POST', + body: backendFormData + }); + + const result = await response.json(); + + if (response.ok) { + this.showSuccess(`${modeLabels[downloadMode]} started! ID: ${result.download_id}`); + e.target.reset(); + } else { + this.showError(result.detail || 'Failed to start download'); + } + + } catch (error) { + console.error('Error:', error); + this.showError('Network error occurred'); + } finally { + this.hideLoading(e.target.querySelector('button')); + } + } + } + + updateProgress(data) { + const download_id = data.download_id || data.id; + console.log('Updating progress for:', download_id, data); + + if (!this.downloads.has(download_id)) { + console.log('Creating new progress card for:', download_id); + this.createProgressCard(download_id, data); + } + + const card = this.downloads.get(download_id); + if (card) { + console.log('Updating existing progress card for:', download_id); + this.updateProgressCard(card, data); + } + + this.showDownloadsList(); + } + + createProgressCard(downloadId, data) { + const card = document.createElement('div'); + card.className = 'progress-card'; + card.id = `progress-${downloadId}`; + + const typeLabel = data.type === 'subreddit' ? 'Subreddit' : 'User'; + const targetName = data.subreddit || data.username || 'Unknown'; + const itemsProcessedInit = Number.isFinite(data.items_processed) ? data.items_processed : (data.data && Number.isFinite(data.data.items_processed) ? data.data.items_processed : 0); + const limitInit = Number.isFinite(data.limit) ? data.limit : (data.data && Number.isFinite(data.data.limit) ? data.data.limit : undefined); + const itemsFoundInit = Number.isFinite(data.items_found) ? data.items_found : (data.data && Number.isFinite(data.data.items_found) ? data.data.items_found : undefined); + const totalInit = (limitInit && limitInit > 0) ? limitInit : itemsFoundInit; + + card.innerHTML = ` +
+
+

${typeLabel}: ${targetName}

+
ID: ${downloadId}
+
Phase: ${this.getPhaseLabel(data.phase || 'queued')}
+
+ Current: ${data.current_item_type === 'submission' ? 'Post' : 'Entry'} ${data.current_item || ''} +
+
+
+
+
${data.status}
+
+
+
+
+
+
+ ${data.progress || 0}% complete + ${Number.isFinite(totalInit) ? `(${itemsProcessedInit}/${totalInit} items)` : ''} + - ${data.message || 'Starting...'} +
+
+ ${this.getProgressDetails(data)} +
+
+ `; + + // Persist basics for retry + card.dataset.subreddit = data.subreddit || ''; + card.dataset.username = data.username || ''; + if (Number.isFinite(totalInit)) { + card.dataset.limit = String(totalInit); + } + + // Attach retry button immediately if already failed due to rate limit + this.maybeAttachRetry(card, data, downloadId); + + this.downloads.set(downloadId, card); + this.downloadsItems.appendChild(card); + } + + updateProgressCard(card, data) { + const statusElement = card.querySelector('.progress-status'); + const progressFill = card.querySelector('.progress-fill'); + const progressText = card.querySelector('.progress-text'); + const phaseElement = card.querySelector('.progress-phase'); + const downloadId = data.download_id || data.id; + const currentItemElement = card.querySelector(`#current-item-${downloadId}`); + const progressDetailsElement = card.querySelector(`#progress-details-${downloadId}`); + + // Update status + statusElement.textContent = data.status; + statusElement.className = `progress-status status-${data.status}`; + + // Update phase + if (phaseElement) { + phaseElement.textContent = `Phase: ${this.getPhaseLabel(data.phase || 'queued')}`; + } + + // Update current item + if (currentItemElement) { + if (data.current_item) { + const itemTypeLabel = data.current_item_type === 'submission' ? 'Post' : 'Entry'; + currentItemElement.textContent = `Current: ${itemTypeLabel} ${data.current_item}`; + currentItemElement.style.display = 'block'; + } else { + currentItemElement.style.display = 'none'; + } + } + + // Update progress bar + if (progressFill) { + progressFill.style.width = `${data.progress || 0}%`; + } + + // Update progress text + if (progressText) { + const itemsProcessed = Number.isFinite(data.items_processed) ? data.items_processed : (data.data && Number.isFinite(data.data.items_processed) ? data.data.items_processed : 0); + const limit = Number.isFinite(data.limit) ? data.limit : (data.data && Number.isFinite(data.data.limit) ? data.data.limit : undefined); + const itemsFound = Number.isFinite(data.items_found) ? data.items_found : (data.data && Number.isFinite(data.data.items_found) ? data.data.items_found : undefined); + const totalItems = (limit && limit > 0) ? limit : itemsFound; + + let progressTextContent = `${data.progress || 0}% complete`; + if (Number.isFinite(totalItems)) { + progressTextContent += ` (${itemsProcessed}/${totalItems} items)`; + } + progressTextContent += ` - ${data.message || 'Processing...'}`; + progressText.textContent = progressTextContent; + } + + // Update progress details + if (progressDetailsElement) { + progressDetailsElement.innerHTML = this.getProgressDetails(data); + } + + // Update card class for animations + card.className = `progress-card ${data.status}`; + + // Persist latest basics for retry + if (data.subreddit) card.dataset.subreddit = data.subreddit; + if (data.username) card.dataset.username = data.username; + if (Number.isFinite(data.limit)) { + card.dataset.limit = String(data.limit); + } else if (data.data && Number.isFinite(data.data.limit)) { + card.dataset.limit = String(data.data.limit); + } + + // Attach retry button if applicable + this.maybeAttachRetry(card, data, downloadId); + } + + getPhaseLabel(phase) { + const phaseLabels = { + 'queued': 'Queued', + 'fetching_submissions': 'Fetching Posts', + 'preparing_download': 'Preparing Download', + 'downloading_submission': 'Downloading Post', + 'writing_file': 'Writing Files', + 'writing_entry': 'Writing Entry', + 'calculating_hashes': 'Calculating Hashes', + 'completed': 'Completed', + 'failed': 'Failed', + 'cancelled': 'Cancelled' + }; + return phaseLabels[phase] || phase.charAt(0).toUpperCase() + phase.slice(1); + } + + getProgressDetails(data) { + let details = ''; + + if (data.current_subreddit) { + details += `
Subreddit: ${data.current_subreddit}
`; + } + + if (data.file_count) { + details += `
Files to Hash: ${data.file_count}
`; + } + + if (data.current_file) { + const shortPath = data.current_file.length > 50 + ? '...' + data.current_file.slice(-47) + : data.current_file; + details += `
Writing: ${shortPath}
`; + } + + return details; + } + + showDownloadsList() { + if (this.downloads.size > 0) { + this.progressContainer.style.display = 'none'; + this.downloadsList.style.display = 'block'; + } else { + this.progressContainer.style.display = 'flex'; + this.downloadsList.style.display = 'none'; + } + } + + validateInput(input) { + const isValid = input.value.trim().length > 0; + input.style.borderColor = isValid ? '#28a745' : '#dc3545'; + return isValid; + } + + validateSubreddit(subreddit) { + return subreddit.length > 0 && subreddit.match(/^[a-zA-Z0-9_]+$/); + } + + validateUsername(username) { + return username.length > 0 && username.match(/^[a-zA-Z0-9_-]+$/); + } + + showLoading(button) { + if (button) { + button.disabled = true; + button.textContent = 'Processing...'; + } + } + + hideLoading(button) { + if (button) { + button.disabled = false; + button.textContent = button.classList.contains('btn-primary') ? 'Start Download' : 'Start Download'; + } + } + + showSuccess(message) { + this.showNotification(message, 'success'); + } + + showError(message) { + this.showNotification(message, 'error'); + } + + showNotification(message, type) { + // Create notification element + const notification = document.createElement('div'); + notification.className = `notification notification-${type}`; + notification.textContent = message; + + // Style the notification + Object.assign(notification.style, { + position: 'fixed', + top: '20px', + right: '20px', + padding: '15px 20px', + borderRadius: '8px', + color: 'white', + fontWeight: '600', + zIndex: '1000', + opacity: '0', + transform: 'translateY(-20px)', + transition: 'all 0.3s ease', + backgroundColor: type === 'success' ? '#28a745' : '#dc3545' + }); + + document.body.appendChild(notification); + + // Animate in + setTimeout(() => { + notification.style.opacity = '1'; + notification.style.transform = 'translateY(0)'; + }, 100); + + // Remove after 5 seconds + setTimeout(() => { + notification.style.opacity = '0'; + notification.style.transform = 'translateY(-20px)'; + setTimeout(() => { + document.body.removeChild(notification); + }, 300); + }, 5000); + } + + updateWSStatus(status) { + if (this.wsStatus) { + this.wsStatus.textContent = status; + this.wsStatus.className = `status-value status-${status.toLowerCase()}`; + } + } + + async updateStatus() { + // try { + // // Update server time + // this.serverTime.textContent = new Date().toLocaleTimeString(); + + // // Check BDFR status + // const response = await fetch('/api/bdfr/status'); + // const status = await response.json(); + + // if (this.bdfrStatus) { + // this.bdfrStatus.textContent = status.bdfr_available ? 'Online' : 'Offline'; + // this.bdfrStatus.className = `status-value status-${status.bdfr_available ? 'online' : 'offline'}`; + // } + + // } catch (error) { + // console.error('Failed to update status:', error); + // } + } + + startStatusPolling() { + // Update status every 30 seconds + setInterval(() => { + this.updateStatus(); + }, 30000); + + // Initial update + this.updateStatus(); + } + + async clearCompletedDownloads() { + const completedCards = Array.from(this.downloadsItems.querySelectorAll('.progress-card.completed')); + + if (completedCards.length === 0) { + this.showNotification('No completed downloads to clear', 'info'); + return; + } + + if (confirm(`Clear ${completedCards.length} completed download(s)?`)) { + completedCards.forEach(card => { + const downloadId = card.id.replace('progress-', ''); + this.downloads.delete(downloadId); + card.remove(); + }); + + this.showDownloadsList(); + this.showSuccess('Completed downloads cleared'); + } + } + + + // Retry helpers + isRateLimited(data) { + const msg = (data && data.message) ? String(data.message) : ''; + const ex = (data && data.data && data.data.exception) ? String(data.data.exception) : ''; + const phase = data && data.data && data.data.phase; + return phase === 'rate_limited' || msg.includes('429') || ex.includes('429'); + } + + maybeAttachRetry(card, data, downloadId) { + try { + const controls = card.querySelector('.progress-controls'); + if (!controls) return; + + const existing = controls.querySelector('.btn-retry'); + + // Detect current event as rate-limited and persist this state on the card + const detected = this.isRateLimited(data); + if (detected) { + card.dataset.rateLimited = 'true'; + } + const persisted = card.dataset.rateLimited === 'true'; + const shouldShow = data.status === 'failed' && (detected || persisted); + + // Remove if no longer applicable; otherwise ensure present + if (!shouldShow) { + if (existing) existing.remove(); + return; + } + + if (!existing) { + const btn = document.createElement('button'); + btn.className = 'btn-retry'; + btn.textContent = 'Retry'; + btn.title = 'Rate limited (429). Retry now.'; + btn.onclick = () => this.retryDownload(downloadId); + controls.appendChild(btn); + } + } catch (e) { + console.warn('Failed to attach retry button', e); + } + } + + async retryDownload(downloadId) { + try { + const card = this.downloads.get(downloadId) || document.getElementById(`progress-${downloadId}`); + if (!card) { + this.showError(`Cannot retry; card not found for ${downloadId}`); + return; + } + + const subreddit = card.dataset.subreddit || ''; + const username = card.dataset.username || ''; + const limitStr = card.dataset.limit || ''; + const limit = parseInt(limitStr, 10); + const hasLimit = Number.isFinite(limit); + + let endpoint = ''; + const formData = new FormData(); + + if (subreddit) { + endpoint = '/api/download/subreddit'; + formData.append('subreddit', subreddit); + formData.append('limit', hasLimit ? String(limit) : '10'); + formData.append('sort', 'hot'); + // Enable no_dupes for retry to avoid re-downloading same files + formData.append('no_dupes', 'true'); + if (this.authState) formData.append('auth_state', this.authState); + } else if (username) { + endpoint = '/api/download/user'; + formData.append('username', username); + formData.append('limit', hasLimit ? String(limit) : '10'); + formData.append('submitted', 'true'); + // Enable no_dupes for retry to avoid re-downloading same files + formData.append('no_dupes', 'true'); + if (this.authState) formData.append('auth_state', this.authState); + } else { + this.showError('Cannot determine original request (subreddit/user) for retry'); + return; + } + + // Remove old failed card before retrying + if (card.remove) card.remove(); + this.downloads.delete(downloadId); + + this.showNotification(`Retrying ${subreddit ? `r/${subreddit}` : `u/${username}`}...`, 'info'); + + const response = await fetch(endpoint, { method: 'POST', body: formData }); + const result = await response.json(); + + if (response.ok) { + this.showSuccess(`Retry started: ${result.download_id}`); + // New download card will appear via websocket updates + } else { + this.showError(result.detail || 'Failed to start retry'); + } + } catch (error) { + console.error('Retry error:', error); + this.showError('Retry failed'); + } + } + + // Authentication methods + async checkAuthentication() { + try { + // Use stored auth state if available + const stateToCheck = this.authState || this.getStoredAuthState(); + + const response = await fetch(`/api/auth/status${stateToCheck ? `?state=${stateToCheck}` : ''}`); + const authData = await response.json(); + + console.log('Auth check:', { + stateUsed: stateToCheck, + authenticated: authData.authenticated, + message: authData.message + }); + + this.authenticated = authData.authenticated; + + // If authenticated, store the state for future use + if (authData.authenticated && stateToCheck) { + this.authState = stateToCheck; + this.storeAuthState(stateToCheck); + } + + this.updateAuthDisplay(authData); + + } catch (error) { + console.error('Failed to check authentication:', error); + } + } + + getStoredAuthState() { + // Try to get stored state from sessionStorage + return sessionStorage.getItem('bdfr_auth_state'); + } + + storeAuthState(state) { + // Store state in sessionStorage for persistence + sessionStorage.setItem('bdfr_auth_state', state); + } + + updateAuthDisplay(authData) { + if (authData.authenticated) { + this.authSection.style.display = 'block'; + this.authStatus.textContent = '🟢 Connected'; + this.authStatus.className = 'auth-status-indicator connected'; + this.loginBtn.style.display = 'none'; + this.logoutBtn.style.display = 'inline-block'; + + // Show Reddit username when authenticated + if (this.authUser) this.authUser.textContent = (authData.username || '').toString() || '-'; + + // Update forms with auth state + if (this.authStateInput) this.authStateInput.value = this.authState || ''; + if (this.userAuthStateInput) this.userAuthStateInput.value = this.authState || ''; + + } else { + this.authSection.style.display = 'block'; + this.authStatus.textContent = '🔴 Not Connected'; + this.authStatus.className = 'auth-status-indicator disconnected'; + this.loginBtn.style.display = 'inline-block'; + this.logoutBtn.style.display = 'none'; + + // Reset Reddit username display + if (this.authUser) this.authUser.textContent = '-'; + + // Clear auth state from forms + if (this.authStateInput) this.authStateInput.value = ''; + if (this.userAuthStateInput) this.userAuthStateInput.value = ''; + } + } + + async handleLogin() { + try { + // Get OAuth2 authorization URL + const response = await fetch('/auth/login'); + const authData = await response.json(); + + if (response.ok) { + // Store state for later use + this.authState = authData.state; + this.storeAuthState(this.authState); + + // Redirect to Reddit OAuth2 + window.location.href = authData.authorization_url; + } else { + this.showError(authData.detail || 'Failed to initiate login'); + } + + } catch (error) { + console.error('Login error:', error); + this.showError('Failed to connect to Reddit'); + } + } + + async handleLogout() { + if (!this.authState) { + this.showError('No active session to logout'); + return; + } + + try { + const formData = new FormData(); + formData.append('state', this.authState); + + const response = await fetch('/auth/logout', { + method: 'POST', + body: formData + }); + + if (response.ok) { + this.authState = null; + this.authenticated = false; + // Clear stored auth state + sessionStorage.removeItem('bdfr_auth_state'); + this.updateAuthDisplay({ authenticated: false }); + this.showSuccess('Successfully logged out'); + } else { + const error = await response.json(); + this.showError(error.detail || 'Logout failed'); + } + + } catch (error) { + console.error('Logout error:', error); + this.showError('Logout failed'); + } + } + + // Handle OAuth2 callback + handleOAuth2Callback() { + const urlParams = new URLSearchParams(window.location.search); + const code = urlParams.get('code'); + const state = urlParams.get('state'); + const error = urlParams.get('error'); + + if (error) { + this.showError(`OAuth2 error: ${error}`); + return; + } + + if (code && state) { + this.completeOAuth2Flow(code, state); + } + } + + async completeOAuth2Flow(code, state) { + try { + const formData = new FormData(); + formData.append('code', code); + formData.append('state', state); + + const response = await fetch('/auth/callback', { + method: 'POST', + body: formData + }); + + if (response.ok) { + // Check if response is HTML (success page) or JSON + const contentType = response.headers.get('content-type'); + + if (contentType && contentType.includes('text/html')) { + // It's the success page, just show success message + this.showSuccess('Successfully authenticated with Reddit!'); + this.authenticated = true; + + // Store auth state for persistence (use a default since we don't have it from HTML response) + // The actual state will be retrieved when checking auth status + this.storeAuthState('active_session'); + + this.updateAuthDisplay({ authenticated: true }); + + // Clean URL + window.history.replaceState({}, document.title, window.location.pathname); + } else { + // It's JSON response (fallback) + const result = await response.json(); + this.authState = result.tokens ? result.tokens.state : null; + this.authenticated = true; + this.updateAuthDisplay({ authenticated: true }); + + // Update user display if available + if (result.user && this.authUser) { + this.authUser.textContent = result.user.name; + } + + this.showSuccess('Successfully authenticated with Reddit!'); + window.history.replaceState({}, document.title, window.location.pathname); + } + } else { + // Try to parse as JSON, but handle HTML error pages gracefully + try { + const result = await response.json(); + this.showError(result.detail || 'Authentication failed'); + } catch (parseError) { + // If it's not JSON, it might be an HTML error page + this.showError('Authentication failed - please check your OAuth configuration'); + } + } + + } catch (error) { + console.error('OAuth2 completion error:', error); + this.showError('Authentication failed'); + } + } +} + +// Initialize the application when DOM is loaded +document.addEventListener('DOMContentLoaded', () => { + window.bdfrApp = new BDFRApp(); + + // Check for OAuth2 callback parameters + const urlParams = new URLSearchParams(window.location.search); + if (urlParams.has('code') || urlParams.has('error')) { + window.bdfrApp.handleOAuth2Callback(); + } + + // Check for authentication success indicator + if (urlParams.has('authenticated')) { + window.bdfrApp.showSuccess('Successfully authenticated with Reddit!'); + window.bdfrApp.authenticated = true; + window.bdfrApp.updateAuthDisplay({ authenticated: true }); + window.history.replaceState({}, document.title, window.location.pathname); + } + + // Check for authentication error indicator + if (urlParams.has('auth_error')) { + window.bdfrApp.showError('Authentication failed - please check your OAuth configuration'); + window.history.replaceState({}, document.title, window.location.pathname); + } +}); + +// Handle page visibility changes +document.addEventListener('visibilitychange', () => { + if (document.visibilityState === 'visible' && window.bdfrApp) { + window.bdfrApp.updateStatus(); + window.bdfrApp.checkAuthentication(); + } +}); \ No newline at end of file diff --git a/web_interface/templates/auth_error.html b/web_interface/templates/auth_error.html new file mode 100644 index 0000000..6ae4c60 --- /dev/null +++ b/web_interface/templates/auth_error.html @@ -0,0 +1,86 @@ + + + + + + Authentication Failed - BDFR Web Interface + + + +
+
❌
+

Authentication Failed

+

+ There was an error during the authentication process. This might be due to: +

+
    +
  • Invalid or expired authorization code
    • +
  • Mismatched redirect URI configuration
    • +
  • Reddit OAuth app not properly configured
    • +
+
{{ error }}
+ Return to Main Page +
+ + \ No newline at end of file diff --git a/web_interface/templates/auth_success.html b/web_interface/templates/auth_success.html new file mode 100644 index 0000000..bf94d03 --- /dev/null +++ b/web_interface/templates/auth_success.html @@ -0,0 +1,82 @@ + + + + + + Authentication Successful - BDFR Web Interface + + + +
+
✅
+

Authentication Successful!

+

+ You have successfully authenticated with Reddit. You can now use all features of the BDFR Web Interface. +

+ +

+ Redirecting you back to the main interface... +

+
+ + + + \ No newline at end of file diff --git a/web_interface/templates/index.html b/web_interface/templates/index.html new file mode 100644 index 0000000..dac1c3e --- /dev/null +++ b/web_interface/templates/index.html @@ -0,0 +1,209 @@ + + + + + + BDFR Web Interface + + + +
+
+

Bulk Downloader for Reddit

+

Web Interface

+ + + +
+ +
+ +
+
+
+

📥 Download Reddit Content

+
+ +
+

🎯 Download Mode

+
+ + + +
+
+ + +
+

📍 Source Type

+
+ + +
+
+ + +
+ + + Enter subreddit name without 'r/' +
+ + +
+
+ + + Max posts to process (1-1000) +
+ +
+ + +
+
+ +
+
+ + +
+ +
+ + + Minimum upvotes +
+
+ + +
+

⚙️ Advanced Options

+
+ + + + + +
+
+ + + +
+
+
+
+ + +
+

📊 Download Progress

+
+
+
+
📥
+

No active downloads

+

Start a download above to see progress here.

+
+
+
+ + + +
+ + +
+
+

System Status

+
+ BDFR Status: + Checking... +
+
+ WebSocket: + Disconnected +
+
+
+
+ +
+

© 2024 BDFR Web Interface. Powered by FastAPI.

+
+
+ + + + \ No newline at end of file