1007 lines
40 KiB
Python
1007 lines
40 KiB
Python
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException, Form, File, UploadFile, Query, status
|
|
from fastapi.responses import HTMLResponse, FileResponse, RedirectResponse
|
|
from fastapi.staticfiles import StaticFiles
|
|
from fastapi.templating import Jinja2Templates
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from fastapi import Request
|
|
import json
|
|
import asyncio
|
|
import os
|
|
import configparser
|
|
import logging
|
|
|
|
# Load environment variables from .env file if available
|
|
try:
|
|
from dotenv import load_dotenv
|
|
load_dotenv()
|
|
except ImportError:
|
|
# python-dotenv not installed, use os.environ directly
|
|
pass
|
|
from typing import List, Dict, Any, Optional
|
|
from datetime import datetime
|
|
from urllib.parse import urlencode
|
|
|
|
# Configure logging
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Import authentication module
|
|
from .auth import init_oauth_manager, get_oauth_manager
|
|
|
|
# Import scheduled tasks modules
|
|
from .database import init_database
|
|
from .scheduler import start_scheduler, stop_scheduler
|
|
from .scheduled_tasks import router as scheduled_tasks_router
|
|
|
|
# Import BDFR API layer
|
|
import sys
|
|
import os
|
|
# Add the parent directory (BDFR root) to Python path
|
|
bdfr_root = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 'bdfr')
|
|
sys.path.insert(0, bdfr_root)
|
|
|
|
# Also add the current bdfr directory to path
|
|
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), ''))
|
|
|
|
try:
|
|
from bdfr.api import BDFRManager, ProgressEvent, ProgressCallback, get_bdfr_manager
|
|
BDFR_AVAILABLE = True
|
|
except ImportError as e:
|
|
logger.warning(f"Failed to import BDFR API: {e}. BDFR modules may not be available.")
|
|
BDFR_AVAILABLE = False
|
|
# Create a mock BDFRManager for when BDFR is not available
|
|
class MockBDFRManager:
|
|
def __init__(self, *args, **kwargs):
|
|
pass
|
|
def download_subreddit(self, *args, **kwargs):
|
|
raise NotImplementedError("BDFR not available")
|
|
def download_user(self, *args, **kwargs):
|
|
raise NotImplementedError("BDFR not available")
|
|
def get_download_status(self, *args, **kwargs):
|
|
return None
|
|
def cancel_download(self, *args, **kwargs):
|
|
return False
|
|
|
|
class MockProgressCallback:
|
|
pass
|
|
|
|
class MockProgressEvent:
|
|
def __init__(self, *args, **kwargs):
|
|
pass
|
|
|
|
BDFRManager = MockBDFRManager
|
|
ProgressEvent = MockProgressEvent
|
|
ProgressCallback = MockProgressCallback
|
|
def get_bdfr_manager(*args, **kwargs):
|
|
return MockBDFRManager()
|
|
|
|
app = FastAPI(title="BDFR Web Interface", version="1.0.0")
|
|
|
|
# Application lifecycle events
|
|
@app.on_event("startup")
|
|
async def startup_event():
|
|
"""Initialize services on application startup"""
|
|
try:
|
|
logger.info("Starting up BDFR Web Interface...")
|
|
|
|
# Initialize database
|
|
init_database()
|
|
logger.info("Database initialized")
|
|
|
|
# Start scheduler
|
|
start_scheduler()
|
|
logger.info("Scheduler started")
|
|
|
|
logger.info("Startup complete!")
|
|
except Exception as e:
|
|
logger.error(f"Startup error: {e}", exc_info=True)
|
|
raise
|
|
|
|
@app.on_event("shutdown")
|
|
async def shutdown_event():
|
|
"""Cleanup on application shutdown"""
|
|
try:
|
|
logger.info("Shutting down BDFR Web Interface...")
|
|
|
|
# Stop scheduler
|
|
stop_scheduler()
|
|
logger.info("Scheduler stopped")
|
|
|
|
# Stop task queue
|
|
from .task_queue import task_queue
|
|
await task_queue.stop()
|
|
logger.info("Task queue stopped")
|
|
|
|
logger.info("Shutdown complete!")
|
|
except Exception as e:
|
|
logger.error(f"Shutdown error: {e}", exc_info=True)
|
|
|
|
# Initialize OAuth2 manager
|
|
def init_oauth():
|
|
"""Initialize OAuth2 manager with credentials from environment or BDFR config"""
|
|
try:
|
|
# Try to get credentials from environment variables first
|
|
client_id = os.getenv("BDFR_CLIENT_ID")
|
|
client_secret = os.getenv("BDFR_CLIENT_SECRET")
|
|
|
|
if client_id and client_secret:
|
|
# Use environment credentials
|
|
logger.info("Using OAuth credentials from environment variables")
|
|
scopes = ["identity", "history", "read", "save", "mysubreddits"]
|
|
else:
|
|
# Fall back to BDFR config file
|
|
logger.info("Using OAuth credentials from BDFR config file")
|
|
config = configparser.ConfigParser()
|
|
config.read("../bdfr/default_config.cfg")
|
|
|
|
client_id = config.get("DEFAULT", "client_id")
|
|
client_secret = config.get("DEFAULT", "client_secret")
|
|
scopes_str = config.get("DEFAULT", "scopes", fallback="identity,read")
|
|
|
|
# Parse scopes
|
|
scopes = [scope.strip() for scope in scopes_str.split(",")]
|
|
|
|
init_oauth_manager(client_id, client_secret, scopes)
|
|
logger.info("OAuth2 manager initialized successfully")
|
|
except Exception as e:
|
|
logger.error(f"Failed to initialize OAuth2 manager: {e}")
|
|
# Use default credentials if config fails
|
|
init_oauth_manager("U-6gk4ZCh3IeNQ", "7CZHY6AmKweZME5s50SfDGylaPg")
|
|
|
|
# Initialize OAuth2 on startup
|
|
# Use a configurable redirect URI - this should match your Reddit OAuth app settings
|
|
redirect_uri = os.getenv("BDFR_REDIRECT_URI", "http://localhost:8000/auth/callback")
|
|
init_oauth()
|
|
|
|
# Enable CORS
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=["*"],
|
|
allow_credentials=True,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
# Mount static files
|
|
import os
|
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
|
static_dir = os.path.join(current_dir, "..", "static")
|
|
template_dir = os.path.join(current_dir, "..", "templates")
|
|
|
|
# Ensure directories exist
|
|
os.makedirs(static_dir, exist_ok=True)
|
|
os.makedirs(template_dir, exist_ok=True)
|
|
|
|
app.mount("/static", StaticFiles(directory=static_dir), name="static")
|
|
templates = Jinja2Templates(directory=template_dir)
|
|
|
|
# Include scheduled tasks router
|
|
app.include_router(scheduled_tasks_router)
|
|
|
|
# WebSocket connection manager
|
|
class ConnectionManager:
|
|
def __init__(self):
|
|
self.active_connections: List[WebSocket] = []
|
|
|
|
async def connect(self, websocket: WebSocket):
|
|
logger.info(f"[WEBSOCKET] WebSocket connection attempt from {websocket.client}")
|
|
await websocket.accept()
|
|
self.active_connections.append(websocket)
|
|
logger.info(f"[WEBSOCKET] New connection established from {websocket.client}. Total connections: {len(self.active_connections)}")
|
|
|
|
def disconnect(self, websocket: WebSocket):
|
|
if websocket in self.active_connections:
|
|
self.active_connections.remove(websocket)
|
|
logger.info(f"[WEBSOCKET] Connection disconnected from {websocket.client}. Total connections: {len(self.active_connections)}")
|
|
|
|
async def send_personal_message(self, message: str, websocket: WebSocket):
|
|
try:
|
|
await websocket.send_text(message)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to send personal message: {e}")
|
|
self.disconnect(websocket)
|
|
|
|
async def broadcast(self, message: str):
|
|
"""Broadcast message to all active connections - simplified version"""
|
|
logger.debug(f"[WEBSOCKET] Broadcasting message to {len(self.active_connections)} connections")
|
|
|
|
if not self.active_connections:
|
|
logger.debug("[WEBSOCKET] No active connections to broadcast to!")
|
|
return
|
|
|
|
# Simple approach: try to send to all connections, remove failed ones immediately
|
|
alive_connections = []
|
|
|
|
for connection in self.active_connections:
|
|
try:
|
|
await connection.send_text(message)
|
|
alive_connections.append(connection)
|
|
logger.debug(f"[WEBSOCKET] Message sent successfully to connection {id(connection)}")
|
|
except Exception as e:
|
|
logger.warning(f"[WEBSOCKET] Removing failed connection {id(connection)}: {e}")
|
|
# Connection failed, don't add it to alive_connections
|
|
|
|
# Update active connections to only include successful ones
|
|
self.active_connections = alive_connections
|
|
logger.debug(f"[WEBSOCKET] Broadcast complete. Active connections: {len(self.active_connections)}")
|
|
|
|
async def is_connection_alive(self, websocket: WebSocket) -> bool:
|
|
"""Check if a WebSocket connection is still alive"""
|
|
try:
|
|
# Try to send a ping frame (this is a WebSocket protocol ping)
|
|
await websocket.ping()
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
async def cleanup_dead_connections(self):
|
|
"""Simplified cleanup - just log the current state"""
|
|
logger.info(f"[WEBSOCKET] Cleanup check: {len(self.active_connections)} active connections")
|
|
|
|
manager = ConnectionManager()
|
|
|
|
# Initialize BDFR Manager
|
|
bdfr_manager = get_bdfr_manager("./downloads")
|
|
|
|
# Store auth token for BDFR manager if available
|
|
_bdfr_auth_token = None
|
|
|
|
# Active downloads tracking (now managed by BDFRManager, but kept for WebSocket compatibility)
|
|
active_downloads = {}
|
|
|
|
# WebSocket-compatible progress callback
|
|
class WebSocketProgressCallback(ProgressCallback if BDFR_AVAILABLE else MockProgressCallback):
|
|
"""Progress callback that sends updates to WebSocket clients"""
|
|
|
|
def __init__(self, download_id: str, connection_manager: ConnectionManager):
|
|
self.download_id = download_id
|
|
self.connection_manager = connection_manager
|
|
|
|
async def on_progress(self, event: ProgressEvent):
|
|
"""Send progress update to WebSocket clients"""
|
|
logger.info(f"[WEBSOCKET-PROGRESS] Received progress event for download {event.download_id}: {event.message} ({event.progress}%)")
|
|
logger.info(f"[WEBSOCKET-PROGRESS] Active connections before broadcast: {len(self.connection_manager.active_connections)}")
|
|
try:
|
|
# Convert BDFR API event to web interface format
|
|
progress_data = {
|
|
"type": "progress",
|
|
"id": event.download_id,
|
|
"download_id": event.download_id,
|
|
"status": "running",
|
|
"message": event.message,
|
|
"progress": event.progress or 0,
|
|
"data": event.data,
|
|
"timestamp": event.timestamp.isoformat()
|
|
}
|
|
|
|
# Update active_downloads for WebSocket compatibility
|
|
# Need to find the web interface download ID that corresponds to this BDFR download ID
|
|
web_download_id = None
|
|
for download_id, download_info in active_downloads.items():
|
|
if download_info.get("bdfr_download_id") == event.download_id:
|
|
web_download_id = download_id
|
|
break
|
|
|
|
if web_download_id:
|
|
# Keep server-side state in sync
|
|
active_downloads[web_download_id]["status"] = "running"
|
|
active_downloads[web_download_id]["progress"] = event.progress or 0
|
|
if "items_processed" in event.data:
|
|
active_downloads[web_download_id]["items_processed"] = event.data["items_processed"]
|
|
if "items_found" in event.data:
|
|
active_downloads[web_download_id]["items_found"] = event.data["items_found"]
|
|
if "current_item" in event.data:
|
|
active_downloads[web_download_id]["current_item"] = event.data["current_item"]
|
|
if "phase" in event.data:
|
|
active_downloads[web_download_id]["phase"] = event.data["phase"]
|
|
|
|
# Unify IDs for frontend to prevent duplicate cards
|
|
progress_data["id"] = web_download_id
|
|
progress_data["download_id"] = web_download_id
|
|
progress_data["bdfr_download_id"] = event.download_id
|
|
progress_data["web_download_id"] = web_download_id
|
|
progress_data["subreddit"] = active_downloads[web_download_id].get("subreddit")
|
|
progress_data["username"] = active_downloads[web_download_id].get("username")
|
|
progress_data["limit"] = active_downloads[web_download_id].get("limit")
|
|
else:
|
|
# No mapping yet; include bdfr id for debugging
|
|
progress_data["bdfr_download_id"] = event.download_id
|
|
progress_data["limit"] = (event.data or {}).get("limit")
|
|
|
|
logger.info(f"[WEBSOCKET-PROGRESS] Broadcasting progress data: {progress_data}")
|
|
logger.info(f"[WEBSOCKET-PROGRESS] About to broadcast progress message for download {event.download_id}")
|
|
await self.connection_manager.broadcast(json.dumps(progress_data))
|
|
logger.info(f"[WEBSOCKET-PROGRESS] Progress message broadcast completed for download {event.download_id}")
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Failed to send progress update for {event.download_id}: {e}")
|
|
|
|
async def on_error(self, event: ProgressEvent):
|
|
"""Send error update to WebSocket clients"""
|
|
logger.info(f"[WEBSOCKET-ERROR] Received error event for download {event.download_id}: {event.message}")
|
|
try:
|
|
error_data = {
|
|
"type": "error",
|
|
"id": event.download_id,
|
|
"download_id": event.download_id,
|
|
"status": "failed",
|
|
"message": event.message,
|
|
"data": event.data,
|
|
"timestamp": event.timestamp.isoformat()
|
|
}
|
|
|
|
# Update active_downloads - need to find the web interface download ID
|
|
# that corresponds to this BDFR download ID
|
|
web_download_id = None
|
|
for download_id, download_info in active_downloads.items():
|
|
if download_info.get("bdfr_download_id") == event.download_id:
|
|
web_download_id = download_id
|
|
break
|
|
|
|
if web_download_id:
|
|
active_downloads[web_download_id]["status"] = "failed"
|
|
active_downloads[web_download_id]["error"] = event.message
|
|
active_downloads[web_download_id]["end_time"] = datetime.now().isoformat()
|
|
# Persist failure phase for status_update broadcasting (e.g., 'rate_limited')
|
|
try:
|
|
if isinstance(event.data, dict) and event.data.get("phase"):
|
|
active_downloads[web_download_id]["phase"] = event.data.get("phase")
|
|
except Exception:
|
|
pass
|
|
logger.info(f"Updated download {web_download_id} to failed status")
|
|
|
|
# Unify IDs for frontend and include mapping
|
|
error_data["id"] = web_download_id
|
|
error_data["download_id"] = web_download_id
|
|
error_data["bdfr_download_id"] = event.download_id
|
|
error_data["web_download_id"] = web_download_id
|
|
error_data["subreddit"] = active_downloads[web_download_id].get("subreddit")
|
|
error_data["username"] = active_downloads[web_download_id].get("username")
|
|
# Also surface phase at top-level for clients that check data.phase or phase
|
|
try:
|
|
if isinstance(event.data, dict) and event.data.get("phase"):
|
|
error_data["phase"] = event.data.get("phase")
|
|
except Exception:
|
|
pass
|
|
else:
|
|
error_data["bdfr_download_id"] = event.download_id
|
|
|
|
await self.connection_manager.broadcast(json.dumps(error_data))
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Failed to send error update for {event.download_id}: {e}")
|
|
logger.warning(f"Error data type: {type(event.data.get('exception'))}")
|
|
|
|
async def on_completed(self, event: ProgressEvent):
|
|
"""Send completion update to WebSocket clients"""
|
|
logger.info(f"[WEBSOCKET-COMPLETED] Received completion event for download {event.download_id}: {event.message}")
|
|
try:
|
|
completed_data = {
|
|
"type": "completed",
|
|
"id": event.download_id,
|
|
"download_id": event.download_id,
|
|
"status": "completed",
|
|
"message": event.message,
|
|
"progress": 100.0,
|
|
"data": event.data,
|
|
"timestamp": event.timestamp.isoformat()
|
|
}
|
|
|
|
# Update active_downloads - need to find the web interface download ID
|
|
# that corresponds to this BDFR download ID
|
|
web_download_id = None
|
|
for download_id, download_info in active_downloads.items():
|
|
if download_info.get("bdfr_download_id") == event.download_id:
|
|
web_download_id = download_id
|
|
break
|
|
|
|
if web_download_id:
|
|
active_downloads[web_download_id]["status"] = "completed"
|
|
active_downloads[web_download_id]["progress"] = 100.0
|
|
active_downloads[web_download_id]["end_time"] = datetime.now().isoformat()
|
|
logger.info(f"Updated download {web_download_id} to completed status")
|
|
|
|
# Unify IDs and include mapping data
|
|
completed_data["id"] = web_download_id
|
|
completed_data["download_id"] = web_download_id
|
|
completed_data["bdfr_download_id"] = event.download_id
|
|
completed_data["web_download_id"] = web_download_id
|
|
completed_data["subreddit"] = active_downloads[web_download_id].get("subreddit")
|
|
completed_data["username"] = active_downloads[web_download_id].get("username")
|
|
else:
|
|
completed_data["bdfr_download_id"] = event.download_id
|
|
|
|
await self.connection_manager.broadcast(json.dumps(completed_data))
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Failed to send completion update for {event.download_id}: {e}")
|
|
|
|
# Helper function to create download with BDFR API
|
|
async def create_download_with_bdfr_api(download_type: str, name: str, **kwargs):
|
|
"""Create a download using the new BDFR API layer"""
|
|
|
|
# Check if BDFR is available
|
|
if not hasattr(bdfr_manager, 'download_subreddit'):
|
|
# BDFR not available, create a mock failed download
|
|
download_id = f"{download_type}_{name.replace('/', '_').replace(' ', '_')}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
|
active_downloads[download_id] = {
|
|
"id": download_id,
|
|
"type": download_type,
|
|
"status": "failed",
|
|
"progress": 0,
|
|
"start_time": datetime.now().isoformat(),
|
|
"authenticated": False,
|
|
"items_processed": 0,
|
|
"items_found": 0,
|
|
"current_item": None,
|
|
"current_item_type": None,
|
|
"phase": "failed",
|
|
"error": "BDFR API not available"
|
|
}
|
|
|
|
# Add type-specific fields
|
|
if download_type in ["subreddit", "archive", "clone"]:
|
|
active_downloads[download_id]["subreddit"] = name
|
|
elif download_type == "user":
|
|
active_downloads[download_id]["username"] = name
|
|
|
|
return download_id
|
|
|
|
# Get global auth token if available
|
|
global _bdfr_auth_token
|
|
auth_token = _bdfr_auth_token
|
|
logger.info(f"[DEBUG] Global auth token available: {auth_token is not None}")
|
|
logger.info(f"[DEBUG] Auth token for BDFR: {auth_token[:10]}..." if auth_token else "None")
|
|
|
|
# Create unique download ID
|
|
download_id = f"{download_type}_{name.replace('/', '_').replace(' ', '_')}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
|
|
|
# Get auth token if provided
|
|
auth_token = None
|
|
logger.info(f"[DEBUG] Checking auth_state: {kwargs.get('auth_state')}")
|
|
if kwargs.get('auth_state'):
|
|
try:
|
|
oauth_manager = get_oauth_manager()
|
|
logger.info(f"[DEBUG] OAuth manager exists: {oauth_manager is not None}")
|
|
auth_token = oauth_manager.get_valid_token(kwargs['auth_state'])
|
|
logger.info(f"[DEBUG] Auth token retrieved for state {kwargs['auth_state']}: {auth_token[:10]}..." if auth_token else "None")
|
|
except Exception as e:
|
|
logger.warning(f"Failed to get auth token: {e}")
|
|
|
|
# Initialize download tracking for WebSocket compatibility
|
|
active_downloads[download_id] = {
|
|
"id": download_id,
|
|
"type": download_type,
|
|
"status": "queued",
|
|
"progress": 0,
|
|
"start_time": datetime.now().isoformat(),
|
|
"limit": kwargs.get('limit'),
|
|
"authenticated": auth_token is not None,
|
|
"items_processed": 0,
|
|
"items_found": 0,
|
|
"current_item": None,
|
|
"current_item_type": None,
|
|
"phase": "queued"
|
|
}
|
|
|
|
# Add type-specific fields
|
|
if download_type in ["subreddit", "archive", "clone"]:
|
|
active_downloads[download_id]["subreddit"] = name
|
|
elif download_type == "user":
|
|
active_downloads[download_id]["username"] = name
|
|
|
|
# Create progress callback
|
|
callback = WebSocketProgressCallback(download_id, manager)
|
|
|
|
# Start download based on type
|
|
if download_type == "subreddit":
|
|
bdfr_download_id = bdfr_manager.download_subreddit(
|
|
name,
|
|
limit=kwargs.get('limit'),
|
|
sort=kwargs.get('sort', 'hot'),
|
|
time_filter=kwargs.get('time_filter', 'all'),
|
|
no_dupes=kwargs.get('no_dupes', False),
|
|
progress_callbacks=[callback]
|
|
)
|
|
elif download_type == "archive":
|
|
# Archive mode - metadata only
|
|
bdfr_download_id = bdfr_manager.archive_subreddit(
|
|
name,
|
|
format_type=kwargs.get('format', 'json'),
|
|
limit=kwargs.get('limit'),
|
|
progress_callbacks=[callback]
|
|
)
|
|
elif download_type == "clone":
|
|
# Clone mode - both download and archive
|
|
bdfr_download_id = bdfr_manager.clone_subreddit(
|
|
name,
|
|
limit=kwargs.get('limit'),
|
|
format_type=kwargs.get('format', 'json'),
|
|
no_dupes=kwargs.get('no_dupes', False),
|
|
progress_callbacks=[callback]
|
|
)
|
|
elif download_type == "user":
|
|
bdfr_download_id = bdfr_manager.download_user(
|
|
name,
|
|
limit=kwargs.get('limit'),
|
|
submitted=kwargs.get('submitted', True),
|
|
upvoted=kwargs.get('upvoted', False),
|
|
saved=kwargs.get('saved', False),
|
|
no_dupes=kwargs.get('no_dupes', False),
|
|
progress_callbacks=[callback]
|
|
)
|
|
else:
|
|
raise ValueError(f"Unsupported download type: {download_type}")
|
|
|
|
# Store BDFR download ID for tracking
|
|
active_downloads[download_id]["bdfr_download_id"] = bdfr_download_id
|
|
|
|
return download_id
|
|
|
|
@app.get("/", response_class=HTMLResponse)
|
|
async def home(request: Request):
|
|
"""Serve the main interface"""
|
|
return templates.TemplateResponse("index.html", {"request": request})
|
|
|
|
@app.get("/health")
|
|
async def health_check():
|
|
"""Health check endpoint"""
|
|
return {"status": "healthy", "timestamp": datetime.now().isoformat()}
|
|
|
|
# OAuth2 Authentication Endpoints
|
|
@app.get("/auth/login")
|
|
async def oauth_login(redirect_uri: str = None):
|
|
"""Initiate OAuth2 login flow"""
|
|
try:
|
|
# Use provided redirect_uri or fall back to configured default
|
|
if redirect_uri is None:
|
|
redirect_uri = os.getenv("BDFR_REDIRECT_URI", "http://localhost:8000/auth/callback")
|
|
|
|
oauth_manager = get_oauth_manager()
|
|
auth_data = oauth_manager.get_authorization_url(redirect_uri)
|
|
|
|
return {
|
|
"authorization_url": auth_data["authorization_url"],
|
|
"state": auth_data["state"],
|
|
"redirect_uri": redirect_uri,
|
|
"message": "Redirect user to the authorization URL"
|
|
}
|
|
except Exception as e:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=f"Failed to initiate OAuth2 login: {str(e)}"
|
|
)
|
|
|
|
@app.get("/auth/callback")
|
|
async def oauth_callback(
|
|
request: Request,
|
|
code: str = Query(..., description="Authorization code from Reddit"),
|
|
state: str = Query(..., description="State parameter for security"),
|
|
error: Optional[str] = Query(None, description="Error from OAuth2 provider")
|
|
):
|
|
"""Handle OAuth2 callback"""
|
|
try:
|
|
if error:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail=f"OAuth2 error: {error}"
|
|
)
|
|
|
|
if not code or not state:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
|
detail="Missing code or state parameter"
|
|
)
|
|
|
|
oauth_manager = get_oauth_manager()
|
|
# Use the same redirect URI that was used for authorization
|
|
redirect_uri = os.getenv("BDFR_REDIRECT_URI", "http://localhost:8000/auth/callback")
|
|
token_data = await oauth_manager.exchange_code_for_token(code, state, redirect_uri)
|
|
|
|
# Get user info
|
|
user_info = await oauth_manager.get_user_info(token_data["access_token"])
|
|
|
|
# Return success page instead of JSON
|
|
return templates.TemplateResponse("auth_success.html", {"request": request})
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"OAuth2 callback error: {str(e)}")
|
|
# Return error page for unexpected errors
|
|
return templates.TemplateResponse("auth_error.html", {
|
|
"request": request,
|
|
"error": "Authentication failed",
|
|
"details": str(e)
|
|
})
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"OAuth2 callback error: {str(e)}")
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail="Authentication failed"
|
|
)
|
|
|
|
@app.get("/auth/status")
|
|
async def auth_status(state: Optional[str] = Query(None)):
|
|
"""Get current authentication status"""
|
|
try:
|
|
oauth_manager = get_oauth_manager()
|
|
status_data = oauth_manager.get_auth_status(state)
|
|
|
|
return {
|
|
"authenticated": status_data["authenticated"],
|
|
"message": status_data.get("message", "Authenticated" if status_data["authenticated"] else "Not authenticated"),
|
|
"expires_at": status_data.get("expires_at"),
|
|
"scopes": status_data.get("scopes"),
|
|
"username": status_data.get("username")
|
|
}
|
|
except Exception as e:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=f"Failed to get auth status: {str(e)}"
|
|
)
|
|
|
|
@app.get("/api/auth/status")
|
|
async def api_auth_status(state: Optional[str] = Query(None)):
|
|
"""Get current authentication status (JSON API)"""
|
|
try:
|
|
oauth_manager = get_oauth_manager()
|
|
|
|
# If no state provided, check if there are any active sessions
|
|
if not state and oauth_manager:
|
|
# Check if there are any stored access tokens
|
|
if oauth_manager.access_tokens:
|
|
# Use the first available state for checking
|
|
state = next(iter(oauth_manager.access_tokens.keys()))
|
|
logger.info(f"No state provided, using first available: {state}")
|
|
|
|
status_data = oauth_manager.get_auth_status(state) if oauth_manager else {"authenticated": False, "message": "No OAuth manager"}
|
|
|
|
# Debug logging
|
|
logger.info(f"Auth status check - State: {state}, Authenticated: {status_data['authenticated']}")
|
|
|
|
# Store auth token for BDFR if authenticated
|
|
global _bdfr_auth_token
|
|
if status_data['authenticated'] and state:
|
|
_bdfr_auth_token = oauth_manager.get_valid_token(state)
|
|
logger.info(f"Stored auth token for BDFR manager: {_bdfr_auth_token is not None}")
|
|
elif not status_data['authenticated']:
|
|
_bdfr_auth_token = None
|
|
logger.info("Cleared auth token for BDFR manager")
|
|
|
|
return {
|
|
"authenticated": status_data["authenticated"],
|
|
"message": status_data.get("message", "Authenticated" if status_data["authenticated"] else "Not authenticated"),
|
|
"expires_at": status_data.get("expires_at", 0),
|
|
"scopes": status_data.get("scopes", []),
|
|
"username": status_data.get("username"),
|
|
"debug_info": {
|
|
"state_provided": state is not None,
|
|
"state_used": state,
|
|
"oauth_manager_exists": oauth_manager is not None,
|
|
"available_states": len(oauth_manager.oauth_states) if oauth_manager else 0,
|
|
"available_tokens": len(oauth_manager.access_tokens) if oauth_manager else 0,
|
|
"bdfr_auth_token_set": _bdfr_auth_token is not None
|
|
}
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"Auth status error: {str(e)}")
|
|
return {
|
|
"authenticated": False,
|
|
"message": f"Error checking auth status: {str(e)}",
|
|
"error": True,
|
|
"debug_info": {
|
|
"error_details": str(e)
|
|
}
|
|
}
|
|
|
|
@app.post("/auth/logout")
|
|
async def auth_logout(state: str = Form(...)):
|
|
"""Logout and revoke OAuth2 session"""
|
|
try:
|
|
oauth_manager = get_oauth_manager()
|
|
oauth_manager.revoke_session(state)
|
|
|
|
return {
|
|
"message": "Successfully logged out",
|
|
"state": state
|
|
}
|
|
except Exception as e:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
detail=f"Logout failed: {str(e)}"
|
|
)
|
|
|
|
@app.post("/api/download/subreddit")
|
|
async def download_subreddit(
|
|
subreddit: str = Form(...),
|
|
limit: int = Form(10),
|
|
sort: str = Form("hot"),
|
|
time_filter: str = Form(""),
|
|
min_score: str = Form(""),
|
|
no_dupes: bool = Form(False),
|
|
simple_check: bool = Form(False),
|
|
make_hard_links: bool = Form(False),
|
|
download_mode: str = Form("download"),
|
|
auth_state: str = Form(None)
|
|
):
|
|
"""Download from subreddit using BDFR API"""
|
|
|
|
# Map download_mode to appropriate BDFR operation
|
|
if download_mode == "archive":
|
|
# Use archive mode
|
|
download_id = await create_download_with_bdfr_api(
|
|
"archive",
|
|
subreddit,
|
|
limit=limit,
|
|
sort=sort,
|
|
time_filter=time_filter or "all",
|
|
format="json",
|
|
simple_check=simple_check,
|
|
auth_state=auth_state
|
|
)
|
|
message = f"Starting archive for r/{subreddit}"
|
|
elif download_mode == "clone":
|
|
# Use clone mode (download + archive)
|
|
download_id = await create_download_with_bdfr_api(
|
|
"clone",
|
|
subreddit,
|
|
limit=limit,
|
|
sort=sort,
|
|
time_filter=time_filter or "all",
|
|
no_dupes=no_dupes,
|
|
simple_check=simple_check,
|
|
format="json",
|
|
auth_state=auth_state
|
|
)
|
|
message = f"Starting clone for r/{subreddit}"
|
|
else:
|
|
# Default: download mode
|
|
download_id = await create_download_with_bdfr_api(
|
|
"subreddit",
|
|
subreddit,
|
|
limit=limit,
|
|
sort=sort,
|
|
time_filter=time_filter or "all",
|
|
no_dupes=no_dupes,
|
|
simple_check=simple_check,
|
|
auth_state=auth_state
|
|
)
|
|
message = f"Starting download for r/{subreddit}"
|
|
|
|
return {
|
|
"download_id": download_id,
|
|
"message": message,
|
|
"mode": download_mode,
|
|
"estimated_time": "2-3 minutes"
|
|
}
|
|
|
|
@app.post("/api/download/user")
|
|
async def download_user(
|
|
username: str = Form(...),
|
|
limit: int = Form(10),
|
|
submitted: bool = Form(True),
|
|
sort: str = Form("hot"),
|
|
time_filter: str = Form(""),
|
|
no_dupes: bool = Form(False),
|
|
simple_check: bool = Form(False),
|
|
make_hard_links: bool = Form(False),
|
|
download_mode: str = Form("download"),
|
|
auth_state: str = Form(None)
|
|
):
|
|
"""Download from user using BDFR API"""
|
|
|
|
# Map download_mode to appropriate BDFR operation
|
|
if download_mode == "archive":
|
|
# Use archive mode
|
|
download_id = await create_download_with_bdfr_api(
|
|
"archive",
|
|
username,
|
|
limit=limit,
|
|
sort=sort,
|
|
time_filter=time_filter or "all",
|
|
format="json",
|
|
submitted=submitted,
|
|
simple_check=simple_check,
|
|
auth_state=auth_state
|
|
)
|
|
message = f"Starting archive for u/{username}"
|
|
elif download_mode == "clone":
|
|
# Use clone mode (download + archive)
|
|
download_id = await create_download_with_bdfr_api(
|
|
"clone",
|
|
username,
|
|
limit=limit,
|
|
sort=sort,
|
|
time_filter=time_filter or "all",
|
|
no_dupes=no_dupes,
|
|
simple_check=simple_check,
|
|
format="json",
|
|
submitted=submitted,
|
|
auth_state=auth_state
|
|
)
|
|
message = f"Starting clone for u/{username}"
|
|
else:
|
|
# Default: download mode
|
|
download_id = await create_download_with_bdfr_api(
|
|
"user",
|
|
username,
|
|
limit=limit,
|
|
sort=sort,
|
|
submitted=submitted,
|
|
no_dupes=no_dupes,
|
|
simple_check=simple_check,
|
|
auth_state=auth_state
|
|
)
|
|
message = f"Starting download for u/{username}"
|
|
|
|
return {
|
|
"download_id": download_id,
|
|
"message": message,
|
|
"mode": download_mode,
|
|
"estimated_time": "1-2 minutes"
|
|
}
|
|
|
|
@app.get("/api/downloads")
|
|
async def get_downloads():
|
|
"""Get all active downloads"""
|
|
return {"downloads": active_downloads}
|
|
|
|
@app.get("/api/downloads/{download_id}")
|
|
async def get_download_status(download_id: str):
|
|
"""Get specific download status"""
|
|
if download_id not in active_downloads:
|
|
raise HTTPException(status_code=404, detail="Download not found")
|
|
return {"download": active_downloads[download_id]}
|
|
|
|
@app.delete("/api/downloads/{download_id}")
|
|
async def cancel_download(download_id: str):
|
|
"""Cancel a download"""
|
|
if download_id not in active_downloads:
|
|
raise HTTPException(status_code=404, detail="Download not found")
|
|
|
|
# Cancel in BDFR manager if we have the BDFR download ID
|
|
if "bdfr_download_id" in active_downloads[download_id]:
|
|
bdfr_manager.cancel_download(active_downloads[download_id]["bdfr_download_id"])
|
|
|
|
active_downloads[download_id]["status"] = "cancelled"
|
|
return {"message": "Download cancelled"}
|
|
|
|
@app.websocket("/ws/progress")
|
|
async def websocket_endpoint(websocket: WebSocket):
|
|
"""WebSocket endpoint for real-time progress updates"""
|
|
logger.info(f"[WEBSOCKET-ENDPOINT] New WebSocket connection attempt from {websocket.client}")
|
|
await manager.connect(websocket)
|
|
logger.info(f"[WEBSOCKET-ENDPOINT] WebSocket connection established successfully")
|
|
logger.info(f"[WEBSOCKET-ENDPOINT] Connection count: {len(manager.active_connections)}")
|
|
|
|
# Track last broadcast state to avoid redundant messages
|
|
last_broadcast_state = {}
|
|
cleanup_counter = 0
|
|
keepalive_counter = 0
|
|
|
|
try:
|
|
while True:
|
|
# Clean up dead connections periodically (every 10 iterations = 20 seconds)
|
|
cleanup_counter += 1
|
|
if cleanup_counter >= 10:
|
|
await manager.cleanup_dead_connections()
|
|
cleanup_counter = 0
|
|
|
|
# Send keepalive ping every 15 iterations (30 seconds) to prevent timeout
|
|
keepalive_counter += 1
|
|
if keepalive_counter >= 15:
|
|
try:
|
|
await websocket.send_json({"type": "keepalive", "timestamp": datetime.now().isoformat()})
|
|
logger.debug(f"[WEBSOCKET-ENDPOINT] Sent keepalive ping")
|
|
except Exception as e:
|
|
logger.warning(f"[WEBSOCKET-ENDPOINT] Failed to send keepalive: {e}")
|
|
break
|
|
keepalive_counter = 0
|
|
|
|
# Check for changes in download states
|
|
has_changes = False
|
|
for download_id, download in list(active_downloads.items()):
|
|
# Only check active downloads
|
|
if download.get("status") in ["running", "queued", "failed"]:
|
|
# Create a state snapshot for comparison
|
|
current_state = {
|
|
"status": download.get("status"),
|
|
"progress": download.get("progress", 0),
|
|
"phase": download.get("phase", "unknown"),
|
|
"items_processed": download.get("items_processed", 0),
|
|
"items_found": download.get("items_found", 0),
|
|
"current_item": download.get("current_item"),
|
|
}
|
|
|
|
# Compare with last broadcast state
|
|
if download_id not in last_broadcast_state or last_broadcast_state[download_id] != current_state:
|
|
has_changes = True
|
|
last_broadcast_state[download_id] = current_state
|
|
|
|
logger.debug(f"[WEBSOCKET-ENDPOINT] State change detected for {download_id}: {download.get('status')}")
|
|
status_message = {
|
|
"type": "status_update",
|
|
"id": download_id,
|
|
"download_id": download_id,
|
|
"status": download["status"],
|
|
"progress": download.get("progress", 0),
|
|
"phase": download.get("phase", "unknown"),
|
|
"items_processed": download.get("items_processed", 0),
|
|
"items_found": download.get("items_found", 0),
|
|
"limit": download.get("limit", 0),
|
|
"current_item": download.get("current_item"),
|
|
"current_item_type": download.get("current_item_type"),
|
|
"current_subreddit": download.get("current_subreddit"),
|
|
"username": download.get("username"),
|
|
"subreddit": download.get("subreddit"),
|
|
"message": f"{download.get('subreddit', download.get('username', 'content'))} - {download['status']}"
|
|
}
|
|
try:
|
|
await manager.broadcast(json.dumps(status_message))
|
|
except Exception as e:
|
|
logger.warning(f"Error broadcasting status update: {e}")
|
|
|
|
if not has_changes:
|
|
logger.debug(f"[WEBSOCKET-ENDPOINT] No state changes detected, skipping broadcast")
|
|
|
|
await asyncio.sleep(2) # Check every 2 seconds
|
|
|
|
except WebSocketDisconnect:
|
|
logger.info(f"[WEBSOCKET-ENDPOINT] WebSocket disconnected normally from {websocket.client}")
|
|
manager.disconnect(websocket)
|
|
except Exception as e:
|
|
logger.error(f"[WEBSOCKET-ENDPOINT] WebSocket error from {websocket.client}: {e}")
|
|
manager.disconnect(websocket)
|
|
|
|
@app.get("/api/bdfr/status")
|
|
async def bdfr_status():
|
|
"""Get BDFR status and available options"""
|
|
return {
|
|
"bdfr_available": BDFR_AVAILABLE,
|
|
"version": "2.0.0",
|
|
"supported_operations": [
|
|
"subreddit_download",
|
|
"user_download",
|
|
"custom_filter"
|
|
],
|
|
"output_formats": ["json", "csv", "xml"]
|
|
}
|
|
|
|
@app.get("/api/websocket/status")
|
|
async def websocket_status():
|
|
"""Get WebSocket connection status for debugging"""
|
|
return {
|
|
"active_connections": len(manager.active_connections),
|
|
"connection_failures": manager.connection_failures,
|
|
"active_downloads": len(active_downloads),
|
|
"websocket_endpoint": "/ws/progress",
|
|
"server_host": "0.0.0.0",
|
|
"server_port": 8000
|
|
}
|
|
|
|
@app.get("/api/oauth/debug")
|
|
async def oauth_debug():
|
|
"""Debug endpoint to check OAuth state"""
|
|
try:
|
|
oauth_manager = get_oauth_manager()
|
|
return {
|
|
"oauth_manager_initialized": oauth_manager is not None,
|
|
"oauth_states_count": len(oauth_manager.oauth_states) if oauth_manager else 0,
|
|
"access_tokens_count": len(oauth_manager.access_tokens) if oauth_manager else 0,
|
|
"refresh_tokens_count": len(oauth_manager.refresh_tokens) if oauth_manager else 0,
|
|
"client_id_configured": oauth_manager.client_id != "U-6gk4ZCh3IeNQ" if oauth_manager else False,
|
|
"redirect_uri": os.getenv("BDFR_REDIRECT_URI", "http://localhost:8000/auth/callback")
|
|
}
|
|
except Exception as e:
|
|
return {
|
|
"error": str(e),
|
|
"oauth_manager_initialized": False
|
|
}
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
uvicorn.run("app.main:app", host="0.0.0.0", port=8000, reload=True) |