feat(UI): added docker image and config
This commit is contained in:
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
Database configuration for scheduled downloads.
|
||||
Uses SQLite with SQLAlchemy ORM for persistent storage.
|
||||
"""
|
||||
|
||||
from sqlalchemy import create_engine, event
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from pathlib import Path
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Database file location - uses /app/data in Docker container
|
||||
# This directory should be mounted as a volume for persistence
|
||||
DATA_DIR = Path(__file__).parent.parent / "data"
|
||||
DB_PATH = DATA_DIR / "scheduled_tasks.db"
|
||||
|
||||
# Ensure data directory exists (will be mounted volume in Docker)
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create engine with proper settings for SQLite in Docker
|
||||
engine = create_engine(
|
||||
f"sqlite:///{DB_PATH}",
|
||||
echo=False,
|
||||
connect_args={
|
||||
"check_same_thread": False, # Allow multi-threaded access
|
||||
"timeout": 30 # Longer timeout for container I/O
|
||||
},
|
||||
pool_pre_ping=True, # Verify connections before using
|
||||
)
|
||||
|
||||
# Enable foreign keys for SQLite
|
||||
@event.listens_for(engine, "connect")
|
||||
def set_sqlite_pragma(dbapi_conn, connection_record):
|
||||
cursor = dbapi_conn.cursor()
|
||||
cursor.execute("PRAGMA foreign_keys=ON")
|
||||
cursor.close()
|
||||
|
||||
# Create session factory
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
# Base class for models
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
def init_database():
|
||||
"""Initialize database - called on container startup"""
|
||||
try:
|
||||
Base.metadata.create_all(bind=engine)
|
||||
logger.info(f"Database initialized at {DB_PATH}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize database: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def get_db():
|
||||
"""
|
||||
Dependency for FastAPI to get database session.
|
||||
|
||||
Usage:
|
||||
@app.get("/endpoint")
|
||||
def endpoint(db: Session = Depends(get_db)):
|
||||
...
|
||||
"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
@@ -28,6 +28,11 @@ logger = logging.getLogger(__name__)
|
||||
# Import authentication module
|
||||
from .auth import init_oauth_manager, get_oauth_manager
|
||||
|
||||
# Import scheduled tasks modules
|
||||
from .database import init_database
|
||||
from .scheduler import start_scheduler, stop_scheduler
|
||||
from .scheduled_tasks import router as scheduled_tasks_router
|
||||
|
||||
# Import BDFR API layer
|
||||
import sys
|
||||
import os
|
||||
@@ -72,6 +77,45 @@ except ImportError as e:
|
||||
|
||||
app = FastAPI(title="BDFR Web Interface", version="1.0.0")
|
||||
|
||||
# Application lifecycle events
|
||||
@app.on_event("startup")
|
||||
async def startup_event():
|
||||
"""Initialize services on application startup"""
|
||||
try:
|
||||
logger.info("Starting up BDFR Web Interface...")
|
||||
|
||||
# Initialize database
|
||||
init_database()
|
||||
logger.info("Database initialized")
|
||||
|
||||
# Start scheduler
|
||||
start_scheduler()
|
||||
logger.info("Scheduler started")
|
||||
|
||||
logger.info("Startup complete!")
|
||||
except Exception as e:
|
||||
logger.error(f"Startup error: {e}", exc_info=True)
|
||||
raise
|
||||
|
||||
@app.on_event("shutdown")
|
||||
async def shutdown_event():
|
||||
"""Cleanup on application shutdown"""
|
||||
try:
|
||||
logger.info("Shutting down BDFR Web Interface...")
|
||||
|
||||
# Stop scheduler
|
||||
stop_scheduler()
|
||||
logger.info("Scheduler stopped")
|
||||
|
||||
# Stop task queue
|
||||
from .task_queue import task_queue
|
||||
await task_queue.stop()
|
||||
logger.info("Task queue stopped")
|
||||
|
||||
logger.info("Shutdown complete!")
|
||||
except Exception as e:
|
||||
logger.error(f"Shutdown error: {e}", exc_info=True)
|
||||
|
||||
# Initialize OAuth2 manager
|
||||
def init_oauth():
|
||||
"""Initialize OAuth2 manager with credentials from environment or BDFR config"""
|
||||
@@ -131,6 +175,9 @@ os.makedirs(template_dir, exist_ok=True)
|
||||
app.mount("/static", StaticFiles(directory=static_dir), name="static")
|
||||
templates = Jinja2Templates(directory=template_dir)
|
||||
|
||||
# Include scheduled tasks router
|
||||
app.include_router(scheduled_tasks_router)
|
||||
|
||||
# WebSocket connection manager
|
||||
class ConnectionManager:
|
||||
def __init__(self):
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""
|
||||
Database models for scheduled downloads.
|
||||
"""
|
||||
|
||||
from sqlalchemy import Column, String, Integer, Boolean, DateTime, Time, ForeignKey, Text
|
||||
from sqlalchemy.orm import relationship
|
||||
from datetime import datetime, time as time_type
|
||||
import uuid
|
||||
import pytz
|
||||
|
||||
from .database import Base
|
||||
|
||||
|
||||
def generate_uuid():
|
||||
"""Generate a UUID string"""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
class ScheduledTask(Base):
|
||||
"""
|
||||
Represents a scheduled download task.
|
||||
Tasks run at a specified time each day.
|
||||
"""
|
||||
__tablename__ = "scheduled_tasks"
|
||||
|
||||
# Primary key
|
||||
id = Column(String(36), primary_key=True, default=generate_uuid)
|
||||
|
||||
# Task identification
|
||||
name = Column(String(255), nullable=False, index=True)
|
||||
enabled = Column(Boolean, default=True, nullable=False, index=True)
|
||||
|
||||
# Download configuration
|
||||
source_type = Column(String(20), nullable=False) # "subreddit" or "user"
|
||||
source_name = Column(String(255), nullable=False, index=True)
|
||||
download_mode = Column(String(20), nullable=False) # "download", "archive", "clone"
|
||||
|
||||
# Filter options
|
||||
limit = Column(Integer, default=25, nullable=False)
|
||||
sort = Column(String(20), default="hot", nullable=False)
|
||||
time_filter = Column(String(20), default="day", nullable=False) # Always "day" for daily tasks
|
||||
min_score = Column(Integer, nullable=True)
|
||||
no_dupes = Column(Boolean, default=True, nullable=False) # Always true for scheduled
|
||||
simple_check = Column(Boolean, default=False, nullable=False)
|
||||
|
||||
# Scheduling
|
||||
schedule_frequency = Column(String(20), default="daily", nullable=False)
|
||||
run_time = Column(Time, nullable=False) # Time of day to run
|
||||
timezone = Column(String(50), default="UTC", nullable=False)
|
||||
|
||||
# Metadata
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(pytz.UTC), nullable=False)
|
||||
updated_at = Column(DateTime, default=lambda: datetime.now(pytz.UTC), onupdate=lambda: datetime.now(pytz.UTC), nullable=False)
|
||||
last_run_at = Column(DateTime, nullable=True, index=True)
|
||||
next_run_at = Column(DateTime, nullable=True, index=True)
|
||||
|
||||
# Authentication
|
||||
auth_state = Column(String(255), nullable=True)
|
||||
|
||||
# Relationships
|
||||
executions = relationship("TaskExecutionHistory", back_populates="task", cascade="all, delete-orphan")
|
||||
|
||||
def to_dict(self):
|
||||
"""Convert to dictionary for JSON serialization"""
|
||||
return {
|
||||
"id": self.id,
|
||||
"name": self.name,
|
||||
"enabled": self.enabled,
|
||||
"source_type": self.source_type,
|
||||
"source_name": self.source_name,
|
||||
"download_mode": self.download_mode,
|
||||
"limit": self.limit,
|
||||
"sort": self.sort,
|
||||
"time_filter": self.time_filter,
|
||||
"min_score": self.min_score,
|
||||
"no_dupes": self.no_dupes,
|
||||
"simple_check": self.simple_check,
|
||||
"schedule_frequency": self.schedule_frequency,
|
||||
"run_time": self.run_time.isoformat() if self.run_time else None,
|
||||
"timezone": self.timezone,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
|
||||
"last_run_at": self.last_run_at.isoformat() if self.last_run_at else None,
|
||||
"next_run_at": self.next_run_at.isoformat() if self.next_run_at else None,
|
||||
"auth_state": self.auth_state
|
||||
}
|
||||
|
||||
|
||||
class TaskExecutionHistory(Base):
|
||||
"""
|
||||
Records each execution of a scheduled task.
|
||||
Tracks success/failure and metrics.
|
||||
"""
|
||||
__tablename__ = "task_execution_history"
|
||||
|
||||
# Primary key
|
||||
id = Column(String(36), primary_key=True, default=generate_uuid)
|
||||
|
||||
# Foreign key to task
|
||||
task_id = Column(String(36), ForeignKey("scheduled_tasks.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
|
||||
# Execution details
|
||||
started_at = Column(DateTime, default=lambda: datetime.now(pytz.UTC), nullable=False, index=True)
|
||||
completed_at = Column(DateTime, nullable=True)
|
||||
status = Column(String(20), default="queued", nullable=False, index=True) # queued, running, success, failed, skipped
|
||||
|
||||
# Results
|
||||
items_found = Column(Integer, default=0, nullable=False)
|
||||
items_downloaded = Column(Integer, default=0, nullable=False)
|
||||
error_message = Column(Text, nullable=True)
|
||||
|
||||
# Link to download
|
||||
download_id = Column(String(255), nullable=True, index=True)
|
||||
|
||||
# Relationships
|
||||
task = relationship("ScheduledTask", back_populates="executions")
|
||||
|
||||
def to_dict(self):
|
||||
"""Convert to dictionary for JSON serialization"""
|
||||
return {
|
||||
"id": self.id,
|
||||
"task_id": self.task_id,
|
||||
"started_at": self.started_at.isoformat() if self.started_at else None,
|
||||
"completed_at": self.completed_at.isoformat() if self.completed_at else None,
|
||||
"status": self.status,
|
||||
"items_found": self.items_found,
|
||||
"items_downloaded": self.items_downloaded,
|
||||
"error_message": self.error_message,
|
||||
"download_id": self.download_id
|
||||
}
|
||||
@@ -0,0 +1,474 @@
|
||||
"""
|
||||
API endpoints and business logic for scheduled tasks management.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy.orm import Session
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional
|
||||
from datetime import datetime, time as time_type
|
||||
import pytz
|
||||
import logging
|
||||
|
||||
from .database import get_db
|
||||
from .models import ScheduledTask, TaskExecutionHistory
|
||||
from .scheduler import schedule_task, unschedule_task, calculate_next_run
|
||||
from .task_queue import task_queue
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Create router
|
||||
router = APIRouter(prefix="/api/scheduled-tasks", tags=["scheduled-tasks"])
|
||||
|
||||
|
||||
# Pydantic models for API
|
||||
class ScheduledTaskCreate(BaseModel):
|
||||
name: str = Field(..., min_length=1, max_length=255)
|
||||
source_type: str = Field(..., pattern="^(subreddit|user)$")
|
||||
source_name: str = Field(..., min_length=1, max_length=255)
|
||||
download_mode: str = Field(..., pattern="^(download|archive|clone)$")
|
||||
limit: int = Field(default=25, ge=1, le=1000)
|
||||
sort: str = Field(default="hot")
|
||||
min_score: Optional[int] = None
|
||||
simple_check: bool = False
|
||||
run_time: str = Field(..., pattern="^([01]?[0-9]|2[0-3]):[0-5][0-9]$") # HH:MM format
|
||||
timezone: str = Field(default="UTC")
|
||||
auth_state: Optional[str] = None
|
||||
|
||||
|
||||
class ScheduledTaskUpdate(BaseModel):
|
||||
name: Optional[str] = Field(None, min_length=1, max_length=255)
|
||||
limit: Optional[int] = Field(None, ge=1, le=1000)
|
||||
sort: Optional[str] = None
|
||||
min_score: Optional[int] = None
|
||||
simple_check: Optional[bool] = None
|
||||
run_time: Optional[str] = Field(None, pattern="^([01]?[0-9]|2[0-3]):[0-5][0-9]$")
|
||||
timezone: Optional[str] = None
|
||||
enabled: Optional[bool] = None
|
||||
|
||||
|
||||
class ScheduledTaskResponse(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
enabled: bool
|
||||
source_type: str
|
||||
source_name: str
|
||||
download_mode: str
|
||||
limit: int
|
||||
sort: str
|
||||
time_filter: str
|
||||
min_score: Optional[int]
|
||||
no_dupes: bool
|
||||
simple_check: bool
|
||||
schedule_frequency: str
|
||||
run_time: str
|
||||
timezone: str
|
||||
created_at: str
|
||||
updated_at: str
|
||||
last_run_at: Optional[str]
|
||||
next_run_at: Optional[str]
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class TaskExecutionResponse(BaseModel):
|
||||
id: str
|
||||
task_id: str
|
||||
started_at: str
|
||||
completed_at: Optional[str]
|
||||
status: str
|
||||
items_found: int
|
||||
items_downloaded: int
|
||||
error_message: Optional[str]
|
||||
download_id: Optional[str]
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
# API Endpoints
|
||||
|
||||
@router.post("", response_model=ScheduledTaskResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def create_scheduled_task(task_data: ScheduledTaskCreate, db: Session = Depends(get_db)):
|
||||
"""
|
||||
Create a new scheduled download task.
|
||||
"""
|
||||
try:
|
||||
# Validate timezone
|
||||
try:
|
||||
pytz.timezone(task_data.timezone)
|
||||
except pytz.exceptions.UnknownTimeZoneError:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid timezone: {task_data.timezone}"
|
||||
)
|
||||
|
||||
# Parse run_time
|
||||
hour, minute = map(int, task_data.run_time.split(':'))
|
||||
run_time_obj = time_type(hour=hour, minute=minute)
|
||||
|
||||
# Create task
|
||||
task = ScheduledTask(
|
||||
name=task_data.name,
|
||||
enabled=True,
|
||||
source_type=task_data.source_type,
|
||||
source_name=task_data.source_name,
|
||||
download_mode=task_data.download_mode,
|
||||
limit=task_data.limit,
|
||||
sort=task_data.sort,
|
||||
time_filter="day", # Always "day" for daily scheduled tasks
|
||||
min_score=task_data.min_score,
|
||||
no_dupes=True, # Always true for scheduled tasks
|
||||
simple_check=task_data.simple_check,
|
||||
schedule_frequency="daily",
|
||||
run_time=run_time_obj,
|
||||
timezone=task_data.timezone,
|
||||
auth_state=task_data.auth_state
|
||||
)
|
||||
|
||||
# Calculate next run time
|
||||
task.next_run_at = calculate_next_run(task)
|
||||
|
||||
# Save to database
|
||||
db.add(task)
|
||||
db.commit()
|
||||
db.refresh(task)
|
||||
|
||||
# Schedule the task
|
||||
schedule_task(task)
|
||||
|
||||
logger.info(f"Created scheduled task {task.id}: {task.name}")
|
||||
|
||||
return task.to_dict()
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create scheduled task: {e}", exc_info=True)
|
||||
db.rollback()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to create scheduled task: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.get("", response_model=List[ScheduledTaskResponse])
|
||||
async def list_scheduled_tasks(db: Session = Depends(get_db)):
|
||||
"""
|
||||
Get all scheduled tasks.
|
||||
"""
|
||||
try:
|
||||
tasks = db.query(ScheduledTask).order_by(ScheduledTask.created_at.desc()).all()
|
||||
return [task.to_dict() for task in tasks]
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list scheduled tasks: {e}", exc_info=True)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to list scheduled tasks"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{task_id}", response_model=ScheduledTaskResponse)
|
||||
async def get_scheduled_task(task_id: str, db: Session = Depends(get_db)):
|
||||
"""
|
||||
Get a specific scheduled task by ID.
|
||||
"""
|
||||
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
|
||||
|
||||
if not task:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Task {task_id} not found"
|
||||
)
|
||||
|
||||
return task.to_dict()
|
||||
|
||||
|
||||
@router.put("/{task_id}", response_model=ScheduledTaskResponse)
|
||||
async def update_scheduled_task(
|
||||
task_id: str,
|
||||
task_data: ScheduledTaskUpdate,
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""
|
||||
Update a scheduled task.
|
||||
"""
|
||||
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
|
||||
|
||||
if not task:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Task {task_id} not found"
|
||||
)
|
||||
|
||||
try:
|
||||
# Update fields
|
||||
if task_data.name is not None:
|
||||
task.name = task_data.name
|
||||
if task_data.limit is not None:
|
||||
task.limit = task_data.limit
|
||||
if task_data.sort is not None:
|
||||
task.sort = task_data.sort
|
||||
if task_data.min_score is not None:
|
||||
task.min_score = task_data.min_score
|
||||
if task_data.simple_check is not None:
|
||||
task.simple_check = task_data.simple_check
|
||||
|
||||
# Handle run_time update
|
||||
reschedule_needed = False
|
||||
if task_data.run_time is not None:
|
||||
hour, minute = map(int, task_data.run_time.split(':'))
|
||||
task.run_time = time_type(hour=hour, minute=minute)
|
||||
reschedule_needed = True
|
||||
|
||||
# Handle timezone update
|
||||
if task_data.timezone is not None:
|
||||
try:
|
||||
pytz.timezone(task_data.timezone)
|
||||
task.timezone = task_data.timezone
|
||||
reschedule_needed = True
|
||||
except pytz.exceptions.UnknownTimeZoneError:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid timezone: {task_data.timezone}"
|
||||
)
|
||||
|
||||
# Handle enabled status
|
||||
if task_data.enabled is not None and task_data.enabled != task.enabled:
|
||||
task.enabled = task_data.enabled
|
||||
reschedule_needed = True
|
||||
|
||||
# Update next_run_at if needed
|
||||
if reschedule_needed:
|
||||
if task.enabled:
|
||||
task.next_run_at = calculate_next_run(task)
|
||||
# Reschedule
|
||||
unschedule_task(task_id)
|
||||
schedule_task(task)
|
||||
else:
|
||||
# Unschedule if disabled
|
||||
unschedule_task(task_id)
|
||||
task.next_run_at = None
|
||||
|
||||
task.updated_at = datetime.now(pytz.UTC)
|
||||
|
||||
db.commit()
|
||||
db.refresh(task)
|
||||
|
||||
logger.info(f"Updated scheduled task {task_id}")
|
||||
|
||||
return task.to_dict()
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update task {task_id}: {e}", exc_info=True)
|
||||
db.rollback()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to update task: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{task_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_scheduled_task(task_id: str, db: Session = Depends(get_db)):
|
||||
"""
|
||||
Delete a scheduled task.
|
||||
"""
|
||||
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
|
||||
|
||||
if not task:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Task {task_id} not found"
|
||||
)
|
||||
|
||||
try:
|
||||
# Unschedule
|
||||
unschedule_task(task_id)
|
||||
|
||||
# Delete from database (cascade will delete execution history)
|
||||
db.delete(task)
|
||||
db.commit()
|
||||
|
||||
logger.info(f"Deleted scheduled task {task_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete task {task_id}: {e}", exc_info=True)
|
||||
db.rollback()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to delete task: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{task_id}/toggle")
|
||||
async def toggle_scheduled_task(task_id: str, db: Session = Depends(get_db)):
|
||||
"""
|
||||
Toggle a task's enabled status.
|
||||
"""
|
||||
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
|
||||
|
||||
if not task:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Task {task_id} not found"
|
||||
)
|
||||
|
||||
try:
|
||||
# Toggle enabled
|
||||
task.enabled = not task.enabled
|
||||
task.updated_at = datetime.now(pytz.UTC)
|
||||
|
||||
if task.enabled:
|
||||
# Re-enable: schedule and calculate next run
|
||||
task.next_run_at = calculate_next_run(task)
|
||||
schedule_task(task)
|
||||
message = "Task enabled"
|
||||
else:
|
||||
# Disable: unschedule
|
||||
unschedule_task(task_id)
|
||||
task.next_run_at = None
|
||||
message = "Task disabled"
|
||||
|
||||
db.commit()
|
||||
|
||||
logger.info(f"Toggled task {task_id}: {message}")
|
||||
|
||||
return {
|
||||
"message": message,
|
||||
"enabled": task.enabled
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to toggle task {task_id}: {e}", exc_info=True)
|
||||
db.rollback()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to toggle task: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{task_id}/run-now")
|
||||
async def run_task_now(task_id: str, db: Session = Depends(get_db)):
|
||||
"""
|
||||
Manually trigger a scheduled task to run now.
|
||||
Adds it to the queue with high priority.
|
||||
"""
|
||||
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
|
||||
|
||||
if not task:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Task {task_id} not found"
|
||||
)
|
||||
|
||||
try:
|
||||
# Add to queue with priority (goes ahead of scheduled tasks)
|
||||
await task_queue.add_task(task_id, priority=1)
|
||||
|
||||
queue_status = task_queue.get_queue_status()
|
||||
|
||||
logger.info(f"Manually queued task {task_id} for immediate execution")
|
||||
|
||||
return {
|
||||
"message": f"Task queued for execution",
|
||||
"task_id": task_id,
|
||||
"queue_position": queue_status['queue_size'],
|
||||
"currently_running": queue_status['current_task'],
|
||||
"status": "queued" if queue_status['current_task'] else "starting"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to queue task {task_id}: {e}", exc_info=True)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to queue task: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{task_id}/history", response_model=List[TaskExecutionResponse])
|
||||
async def get_task_history(task_id: str, limit: int = 10, db: Session = Depends(get_db)):
|
||||
"""
|
||||
Get execution history for a specific task.
|
||||
"""
|
||||
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
|
||||
|
||||
if not task:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Task {task_id} not found"
|
||||
)
|
||||
|
||||
try:
|
||||
executions = db.query(TaskExecutionHistory)\
|
||||
.filter(TaskExecutionHistory.task_id == task_id)\
|
||||
.order_by(TaskExecutionHistory.started_at.desc())\
|
||||
.limit(limit)\
|
||||
.all()
|
||||
|
||||
return [execution.to_dict() for execution in executions]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get history for task {task_id}: {e}", exc_info=True)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to get task history"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/history/recent", response_model=List[TaskExecutionResponse])
|
||||
async def get_recent_history(limit: int = 20, db: Session = Depends(get_db)):
|
||||
"""
|
||||
Get recent execution history across all tasks.
|
||||
"""
|
||||
try:
|
||||
executions = db.query(TaskExecutionHistory)\
|
||||
.order_by(TaskExecutionHistory.started_at.desc())\
|
||||
.limit(limit)\
|
||||
.all()
|
||||
|
||||
return [execution.to_dict() for execution in executions]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get recent history: {e}", exc_info=True)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to get execution history"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/queue/status")
|
||||
async def get_queue_status():
|
||||
"""Get current task queue status"""
|
||||
try:
|
||||
status = task_queue.get_queue_status()
|
||||
|
||||
# Get details of current task if any
|
||||
current_task_info = None
|
||||
if status['current_task']:
|
||||
db = next(get_db())
|
||||
try:
|
||||
task = db.query(ScheduledTask).filter(ScheduledTask.id == status['current_task']).first()
|
||||
if task:
|
||||
current_task_info = {
|
||||
'id': task.id,
|
||||
'name': task.name,
|
||||
'source': f"{task.source_type}/{task.source_name}"
|
||||
}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return {
|
||||
'queue_size': status['queue_size'],
|
||||
'is_processing': status['is_processing'],
|
||||
'current_task': current_task_info
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get queue status: {e}", exc_info=True)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to get queue status"
|
||||
)
|
||||
@@ -0,0 +1,360 @@
|
||||
"""
|
||||
Scheduler service for managing scheduled download tasks.
|
||||
Uses APScheduler with task queue for sequential execution.
|
||||
"""
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
|
||||
from datetime import datetime, time as time_type, timedelta
|
||||
from sqlalchemy.orm import Session
|
||||
import pytz
|
||||
import logging
|
||||
import asyncio
|
||||
|
||||
from .database import SessionLocal, DATA_DIR
|
||||
from .models import ScheduledTask, TaskExecutionHistory
|
||||
from .task_queue import task_queue
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Scheduler job store database (separate from main DB)
|
||||
SCHEDULER_DB_PATH = DATA_DIR / "scheduler_jobs.db"
|
||||
|
||||
# Configure job stores
|
||||
jobstores = {
|
||||
'default': SQLAlchemyJobStore(url=f'sqlite:///{SCHEDULER_DB_PATH}')
|
||||
}
|
||||
|
||||
# Configure scheduler for Docker container
|
||||
scheduler = AsyncIOScheduler(
|
||||
jobstores=jobstores,
|
||||
timezone=pytz.UTC, # Container runs in UTC
|
||||
job_defaults={
|
||||
'coalesce': True, # Combine multiple missed executions into one
|
||||
'max_instances': 1, # Only one instance of each job at a time
|
||||
'misfire_grace_time': 3600 # Allow up to 1 hour late execution
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def queue_scheduled_task(task_id: str):
|
||||
"""
|
||||
Called by scheduler at the configured time.
|
||||
Adds task to queue rather than executing immediately.
|
||||
|
||||
Args:
|
||||
task_id: UUID of the scheduled task
|
||||
"""
|
||||
logger.info(f"Scheduler triggered for task {task_id}, adding to queue")
|
||||
try:
|
||||
await task_queue.add_task(task_id, priority=0) # Normal priority for scheduled tasks
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to queue scheduled task {task_id}: {e}", exc_info=True)
|
||||
|
||||
|
||||
def schedule_task(task: ScheduledTask):
|
||||
"""
|
||||
Schedule a task to be added to the queue at specified time.
|
||||
|
||||
Args:
|
||||
task: ScheduledTask model instance
|
||||
"""
|
||||
if not task.enabled:
|
||||
logger.info(f"Skipping scheduling for disabled task {task.id}")
|
||||
return
|
||||
|
||||
try:
|
||||
# Convert user's timezone to UTC for container execution
|
||||
user_tz = pytz.timezone(task.timezone)
|
||||
|
||||
# Parse time
|
||||
if isinstance(task.run_time, str):
|
||||
hour, minute = map(int, task.run_time.split(':')[:2])
|
||||
else:
|
||||
hour = task.run_time.hour
|
||||
minute = task.run_time.minute
|
||||
|
||||
# Create cron trigger with user's timezone
|
||||
trigger = CronTrigger(
|
||||
hour=hour,
|
||||
minute=minute,
|
||||
timezone=user_tz
|
||||
)
|
||||
|
||||
# Add job to scheduler
|
||||
scheduler.add_job(
|
||||
func=queue_scheduled_task,
|
||||
trigger=trigger,
|
||||
args=[task.id],
|
||||
id=str(task.id),
|
||||
replace_existing=True,
|
||||
name=f"{task.name} ({task.source_type}/{task.source_name})"
|
||||
)
|
||||
|
||||
logger.info(f"Scheduled task {task.id} '{task.name}' for {hour:02d}:{minute:02d} {task.timezone}")
|
||||
|
||||
# Update next_run_at
|
||||
next_run = trigger.get_next_fire_time(None, datetime.now(user_tz))
|
||||
if next_run:
|
||||
db = SessionLocal()
|
||||
try:
|
||||
db_task = db.query(ScheduledTask).filter(ScheduledTask.id == task.id).first()
|
||||
if db_task:
|
||||
db_task.next_run_at = next_run
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to schedule task {task.id}: {e}", exc_info=True)
|
||||
|
||||
|
||||
def unschedule_task(task_id: str):
|
||||
"""
|
||||
Remove a task from the scheduler.
|
||||
|
||||
Args:
|
||||
task_id: UUID of the scheduled task
|
||||
"""
|
||||
try:
|
||||
if scheduler.get_job(task_id):
|
||||
scheduler.remove_job(task_id)
|
||||
logger.info(f"Unscheduled task {task_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to unschedule task {task_id}: {e}", exc_info=True)
|
||||
|
||||
|
||||
def load_scheduled_tasks():
|
||||
"""
|
||||
Load all enabled scheduled tasks from database and schedule them.
|
||||
Called on application startup.
|
||||
"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
tasks = db.query(ScheduledTask).filter(ScheduledTask.enabled == True).all()
|
||||
logger.info(f"Loading {len(tasks)} enabled scheduled tasks")
|
||||
|
||||
for task in tasks:
|
||||
schedule_task(task)
|
||||
|
||||
logger.info(f"Loaded {len(tasks)} scheduled tasks")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load scheduled tasks: {e}", exc_info=True)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def execute_scheduled_task(task_id: str):
|
||||
"""
|
||||
Execute a scheduled download task.
|
||||
This function BLOCKS until the download is complete,
|
||||
ensuring sequential execution.
|
||||
|
||||
Args:
|
||||
task_id: UUID of the scheduled task
|
||||
"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
# Load task from database
|
||||
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
|
||||
|
||||
if not task:
|
||||
logger.error(f"Task {task_id} not found in database")
|
||||
return
|
||||
|
||||
if not task.enabled:
|
||||
logger.info(f"Skipping disabled task {task_id}")
|
||||
# Still record in history that it was skipped
|
||||
execution = TaskExecutionHistory(
|
||||
task_id=task_id,
|
||||
status='skipped',
|
||||
completed_at=datetime.now(pytz.UTC)
|
||||
)
|
||||
db.add(execution)
|
||||
db.commit()
|
||||
return
|
||||
|
||||
logger.info(f"Executing scheduled task {task_id}: {task.source_type}/{task.source_name}")
|
||||
|
||||
# Create execution history record
|
||||
execution = TaskExecutionHistory(
|
||||
task_id=task_id,
|
||||
status='running',
|
||||
started_at=datetime.now(pytz.UTC)
|
||||
)
|
||||
db.add(execution)
|
||||
db.commit()
|
||||
execution_id = execution.id
|
||||
|
||||
try:
|
||||
# Import here to avoid circular imports
|
||||
from .main import create_download_with_bdfr_api, bdfr_manager
|
||||
|
||||
# Build download parameters
|
||||
kwargs = {
|
||||
'limit': task.limit,
|
||||
'sort': task.sort,
|
||||
'time_filter': 'day', # Always "day" for daily scheduled tasks
|
||||
'no_dupes': True, # Always enabled for scheduled tasks
|
||||
'simple_check': task.simple_check,
|
||||
'auth_state': task.auth_state
|
||||
}
|
||||
|
||||
# Create download using existing API
|
||||
download_id = await create_download_with_bdfr_api(
|
||||
download_type=task.source_type,
|
||||
name=task.source_name,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
logger.info(f"Task {task_id} started as download {download_id}")
|
||||
|
||||
# Update execution record with download_id
|
||||
execution = db.query(TaskExecutionHistory).filter(TaskExecutionHistory.id == execution_id).first()
|
||||
if execution:
|
||||
execution.download_id = download_id
|
||||
db.commit()
|
||||
|
||||
# **CRITICAL: Wait for download to complete before returning**
|
||||
await wait_for_download_completion(download_id, bdfr_manager)
|
||||
|
||||
# Check final status
|
||||
download_status = bdfr_manager.get_download_status(download_id)
|
||||
|
||||
# Update execution history
|
||||
execution = db.query(TaskExecutionHistory).filter(TaskExecutionHistory.id == execution_id).first()
|
||||
if execution:
|
||||
if download_status and download_status['status'] == 'completed':
|
||||
execution.status = 'success'
|
||||
execution.items_found = download_status.get('items_found', 0)
|
||||
execution.items_downloaded = download_status.get('items_processed', 0)
|
||||
logger.info(f"Task {task_id} completed successfully")
|
||||
else:
|
||||
execution.status = 'failed'
|
||||
execution.error_message = download_status.get('error', 'Unknown error') if download_status else 'Download status not found'
|
||||
logger.error(f"Task {task_id} failed: {execution.error_message}")
|
||||
|
||||
execution.completed_at = datetime.now(pytz.UTC)
|
||||
db.commit()
|
||||
|
||||
# Update task timestamps
|
||||
task.last_run_at = datetime.now(pytz.UTC)
|
||||
task.next_run_at = calculate_next_run(task)
|
||||
db.commit()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Task {task_id} execution error: {e}", exc_info=True)
|
||||
|
||||
# Update execution history with error
|
||||
execution = db.query(TaskExecutionHistory).filter(TaskExecutionHistory.id == execution_id).first()
|
||||
if execution:
|
||||
execution.status = 'failed'
|
||||
execution.error_message = str(e)
|
||||
execution.completed_at = datetime.now(pytz.UTC)
|
||||
db.commit()
|
||||
|
||||
raise
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def wait_for_download_completion(download_id: str, bdfr_manager, timeout: int = 3600):
|
||||
"""
|
||||
Wait for a download to complete.
|
||||
Polls the download status until it's no longer running.
|
||||
|
||||
Args:
|
||||
download_id: The download to wait for
|
||||
bdfr_manager: BDFRManager instance
|
||||
timeout: Maximum seconds to wait (default 1 hour)
|
||||
"""
|
||||
start_time = datetime.now()
|
||||
check_interval = 5 # Check every 5 seconds
|
||||
|
||||
logger.info(f"Waiting for download {download_id} to complete...")
|
||||
|
||||
while True:
|
||||
# Check if timeout exceeded
|
||||
elapsed = (datetime.now() - start_time).total_seconds()
|
||||
if elapsed > timeout:
|
||||
logger.error(f"Download {download_id} timed out after {timeout}s")
|
||||
raise TimeoutError(f"Download exceeded timeout of {timeout}s")
|
||||
|
||||
# Check download status
|
||||
status = bdfr_manager.get_download_status(download_id)
|
||||
|
||||
if not status:
|
||||
logger.warning(f"Download {download_id} status not found, assuming complete")
|
||||
break
|
||||
|
||||
download_status = status.get('status', 'unknown')
|
||||
|
||||
# Check if download is finished (completed, failed, or cancelled)
|
||||
if download_status in ['completed', 'failed', 'cancelled']:
|
||||
logger.info(f"Download {download_id} finished with status: {download_status}")
|
||||
break
|
||||
|
||||
# Still running, wait before checking again
|
||||
await asyncio.sleep(check_interval)
|
||||
|
||||
|
||||
def calculate_next_run(task: ScheduledTask) -> datetime:
|
||||
"""
|
||||
Calculate the next run time for a task based on its schedule.
|
||||
|
||||
Args:
|
||||
task: ScheduledTask instance
|
||||
|
||||
Returns:
|
||||
Next run datetime in UTC
|
||||
"""
|
||||
user_tz = pytz.timezone(task.timezone)
|
||||
now = datetime.now(user_tz)
|
||||
|
||||
# Parse run time
|
||||
if isinstance(task.run_time, str):
|
||||
hour, minute = map(int, task.run_time.split(':')[:2])
|
||||
else:
|
||||
hour = task.run_time.hour
|
||||
minute = task.run_time.minute
|
||||
|
||||
# Calculate next run
|
||||
next_run = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
|
||||
|
||||
# If time has passed today, schedule for tomorrow
|
||||
if next_run <= now:
|
||||
next_run += timedelta(days=1)
|
||||
|
||||
# Convert to UTC
|
||||
return next_run.astimezone(pytz.UTC)
|
||||
|
||||
|
||||
def start_scheduler():
|
||||
"""Start the scheduler. Called on application startup."""
|
||||
try:
|
||||
if not scheduler.running:
|
||||
scheduler.start()
|
||||
logger.info("Scheduler started")
|
||||
|
||||
# Set execute callback for task queue
|
||||
task_queue.set_execute_callback(execute_scheduled_task)
|
||||
|
||||
# Load existing tasks
|
||||
load_scheduled_tasks()
|
||||
else:
|
||||
logger.info("Scheduler already running")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start scheduler: {e}", exc_info=True)
|
||||
raise
|
||||
|
||||
|
||||
def stop_scheduler():
|
||||
"""Stop the scheduler. Called on application shutdown."""
|
||||
try:
|
||||
if scheduler.running:
|
||||
scheduler.shutdown(wait=True)
|
||||
logger.info("Scheduler stopped")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to stop scheduler: {e}", exc_info=True)
|
||||
@@ -0,0 +1,148 @@
|
||||
"""
|
||||
Task Queue Manager for sequential execution of scheduled downloads.
|
||||
Ensures only one task runs at a time.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Optional, Dict
|
||||
from datetime import datetime
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TaskQueue:
|
||||
"""
|
||||
Manages sequential execution of scheduled download tasks.
|
||||
Ensures only one task runs at a time.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.queue: asyncio.Queue = asyncio.Queue()
|
||||
self.current_task: Optional[str] = None # Current task_id being executed
|
||||
self.is_processing: bool = False
|
||||
self.worker_task: Optional[asyncio.Task] = None
|
||||
self._execute_callback = None
|
||||
|
||||
def set_execute_callback(self, callback):
|
||||
"""
|
||||
Set the callback function to execute tasks.
|
||||
|
||||
Args:
|
||||
callback: Async function that takes task_id and executes it
|
||||
"""
|
||||
self._execute_callback = callback
|
||||
|
||||
async def add_task(self, task_id: str, priority: int = 0):
|
||||
"""
|
||||
Add a task to the queue.
|
||||
|
||||
Args:
|
||||
task_id: UUID of the scheduled task
|
||||
priority: 0 = scheduled (normal), 1 = manual "Run Now" (higher priority)
|
||||
"""
|
||||
task_info = {
|
||||
'task_id': task_id,
|
||||
'priority': priority,
|
||||
'queued_at': datetime.now()
|
||||
}
|
||||
|
||||
# For priority tasks, we need to reorder the queue
|
||||
if priority > 0:
|
||||
# Get all items from queue
|
||||
items = []
|
||||
while not self.queue.empty():
|
||||
try:
|
||||
items.append(await asyncio.wait_for(self.queue.get(), timeout=0.1))
|
||||
except asyncio.TimeoutError:
|
||||
break
|
||||
|
||||
# Add new priority task
|
||||
await self.queue.put(task_info)
|
||||
|
||||
# Re-add other items
|
||||
for item in items:
|
||||
await self.queue.put(item)
|
||||
|
||||
logger.info(f"Priority task {task_id} added to front of queue (queue_size={self.queue.qsize()})")
|
||||
else:
|
||||
await self.queue.put(task_info)
|
||||
logger.info(f"Task {task_id} added to queue (priority={priority}, queue_size={self.queue.qsize()})")
|
||||
|
||||
# Start worker if not already running
|
||||
if not self.is_processing:
|
||||
await self.start_worker()
|
||||
|
||||
async def start_worker(self):
|
||||
"""Start the queue worker if not already running"""
|
||||
if self.worker_task is None or self.worker_task.done():
|
||||
self.worker_task = asyncio.create_task(self._process_queue())
|
||||
logger.info("Queue worker started")
|
||||
|
||||
async def _process_queue(self):
|
||||
"""Process tasks from queue sequentially"""
|
||||
self.is_processing = True
|
||||
logger.info("Queue worker processing started")
|
||||
|
||||
while True:
|
||||
try:
|
||||
# Wait for next task (with timeout to allow graceful shutdown)
|
||||
try:
|
||||
task_info = await asyncio.wait_for(
|
||||
self.queue.get(),
|
||||
timeout=60.0
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
# Check if queue is empty
|
||||
if self.queue.empty():
|
||||
logger.info("Queue empty, worker stopping")
|
||||
break
|
||||
continue
|
||||
|
||||
task_id = task_info['task_id']
|
||||
self.current_task = task_id
|
||||
|
||||
logger.info(f"Executing task {task_id} from queue (queue_size={self.queue.qsize()})")
|
||||
|
||||
# Execute the task (this will block until download completes)
|
||||
try:
|
||||
if self._execute_callback:
|
||||
await self._execute_callback(task_id)
|
||||
logger.info(f"Task {task_id} completed successfully")
|
||||
else:
|
||||
logger.error(f"No execute callback set, cannot run task {task_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Task {task_id} failed: {e}", exc_info=True)
|
||||
finally:
|
||||
self.current_task = None
|
||||
self.queue.task_done()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Queue worker error: {e}", exc_info=True)
|
||||
|
||||
self.is_processing = False
|
||||
logger.info("Queue worker stopped")
|
||||
|
||||
def get_queue_status(self) -> Dict:
|
||||
"""Get current queue status"""
|
||||
return {
|
||||
'current_task': self.current_task,
|
||||
'queue_size': self.queue.qsize(),
|
||||
'is_processing': self.is_processing
|
||||
}
|
||||
|
||||
async def stop(self):
|
||||
"""Stop the queue worker gracefully"""
|
||||
logger.info("Stopping queue worker...")
|
||||
if self.worker_task and not self.worker_task.done():
|
||||
# Wait for current task to complete
|
||||
try:
|
||||
await asyncio.wait_for(self.worker_task, timeout=300) # 5 minute timeout
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("Queue worker did not stop within timeout, cancelling")
|
||||
self.worker_task.cancel()
|
||||
logger.info("Queue worker stopped")
|
||||
|
||||
|
||||
# Global queue instance
|
||||
task_queue = TaskQueue()
|
||||
Reference in New Issue
Block a user