feat(UI): added docker image and config
This commit is contained in:
+112
@@ -0,0 +1,112 @@
|
|||||||
|
# Python
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*$py.class
|
||||||
|
*.so
|
||||||
|
.Python
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
wheels/
|
||||||
|
share/python-wheels/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
MANIFEST
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
.tox/
|
||||||
|
.nox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*.cover
|
||||||
|
*.py,cover
|
||||||
|
.hypothesis/
|
||||||
|
.pytest_cache/
|
||||||
|
cover/
|
||||||
|
|
||||||
|
# Virtual environments
|
||||||
|
env/
|
||||||
|
venv/
|
||||||
|
ENV/
|
||||||
|
env.bak/
|
||||||
|
venv.bak/
|
||||||
|
.venv/
|
||||||
|
|
||||||
|
# IDE
|
||||||
|
.vscode/
|
||||||
|
.idea/
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
*~
|
||||||
|
.DS_Store
|
||||||
|
|
||||||
|
# Git
|
||||||
|
.git/
|
||||||
|
.gitignore
|
||||||
|
.gitattributes
|
||||||
|
.gitmodules
|
||||||
|
|
||||||
|
# Documentation and configs not needed in image
|
||||||
|
*.md
|
||||||
|
!README.md
|
||||||
|
docs/
|
||||||
|
.pre-commit-config.yaml
|
||||||
|
.markdown_style.rb
|
||||||
|
.mdlrc
|
||||||
|
_config.yml
|
||||||
|
SCHEDULED_DOWNLOADS_PLAN.md
|
||||||
|
|
||||||
|
# Test files
|
||||||
|
tests/
|
||||||
|
test_*.py
|
||||||
|
*_test.py
|
||||||
|
conftest.py
|
||||||
|
|
||||||
|
# CI/CD
|
||||||
|
.github/
|
||||||
|
tox.ini
|
||||||
|
|
||||||
|
# Scripts and dev tools
|
||||||
|
scripts/
|
||||||
|
devscripts/
|
||||||
|
|
||||||
|
# Local data and logs (these should be mounted as volumes)
|
||||||
|
web_interface/data/*.db
|
||||||
|
web_interface/data/*.db-*
|
||||||
|
*.log
|
||||||
|
*.sqlite
|
||||||
|
*.sqlite3
|
||||||
|
|
||||||
|
# Environment files (use docker-compose env instead)
|
||||||
|
.env
|
||||||
|
.env.local
|
||||||
|
.env.*.local
|
||||||
|
|
||||||
|
# Docker files (don't need docker in docker)
|
||||||
|
Dockerfile*
|
||||||
|
docker-compose*.yml
|
||||||
|
.dockerignore
|
||||||
|
|
||||||
|
# OS specific
|
||||||
|
Thumbs.db
|
||||||
|
ehthumbs.db
|
||||||
|
Desktop.ini
|
||||||
|
$RECYCLE.BIN/
|
||||||
|
|
||||||
|
# Temporary files
|
||||||
|
*.tmp
|
||||||
|
*.temp
|
||||||
|
*.bak
|
||||||
|
*.backup
|
||||||
+129
@@ -0,0 +1,129 @@
|
|||||||
|
# BDFR Web Interface - Docker Environment Configuration
|
||||||
|
# Copy this file to .env and update the values as needed
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# REDDIT OAUTH CONFIGURATION (REQUIRED for authenticated features)
|
||||||
|
# ============================================================================
|
||||||
|
# Get these credentials from: https://www.reddit.com/prefs/apps
|
||||||
|
#
|
||||||
|
# Steps to get OAuth credentials:
|
||||||
|
# 1. Go to https://www.reddit.com/prefs/apps
|
||||||
|
# 2. Click "Create App" or "Create Another App"
|
||||||
|
# 3. Fill in:
|
||||||
|
# - Name: BDFR Web Interface (or your choice)
|
||||||
|
# - App type: Select "web app"
|
||||||
|
# - Redirect URI: http://localhost:8000/auth/callback
|
||||||
|
# 4. Copy the client ID (under the app name) and client secret
|
||||||
|
|
||||||
|
BDFR_CLIENT_ID=your_client_id_here
|
||||||
|
BDFR_CLIENT_SECRET=your_client_secret_here
|
||||||
|
|
||||||
|
# IMPORTANT: This MUST match your Reddit OAuth app's redirect URI exactly
|
||||||
|
# If you change the port, update this accordingly
|
||||||
|
BDFR_REDIRECT_URI=http://localhost:8000/auth/callback
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# SERVER CONFIGURATION (Optional)
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# Host to bind to (0.0.0.0 for all interfaces, 127.0.0.1 for localhost only)
|
||||||
|
HOST=0.0.0.0
|
||||||
|
|
||||||
|
# Port for the web interface
|
||||||
|
PORT=8000
|
||||||
|
|
||||||
|
# Enable debug mode (true/false) - DO NOT use in production
|
||||||
|
DEBUG=false
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# DOWNLOAD CONFIGURATION (Optional - Docker handles these by default)
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# Directory where downloads will be stored (inside container)
|
||||||
|
# This is mounted from ./downloads on the host
|
||||||
|
BDFR_DOWNLOAD_DIR=/downloads
|
||||||
|
|
||||||
|
# Directory for application data and databases (inside container)
|
||||||
|
# This is mounted from ./data on the host
|
||||||
|
BDFR_DATA_DIR=/app/data
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# BDFR ADVANCED CONFIGURATION (Optional)
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# BDFR configuration directory (inside container)
|
||||||
|
# Default: /app/data/bdfr-config (stored in mounted data volume)
|
||||||
|
# BDFR_CONFIG_DIR=/app/data/bdfr-config
|
||||||
|
|
||||||
|
# Maximum wait time for rate limiting (in seconds)
|
||||||
|
# Default: 120
|
||||||
|
# BDFR_MAX_WAIT_TIME=120
|
||||||
|
|
||||||
|
# Time format for {DATE} in filenames
|
||||||
|
# Default: ISO 8601 format (%Y-%m-%dT%H:%M:%S)
|
||||||
|
# BDFR_TIME_FORMAT=%Y-%m-%dT%H:%M:%S
|
||||||
|
|
||||||
|
# File naming scheme
|
||||||
|
# Available variables: {REDDITOR}, {SUBREDDIT}, {POSTID}, {UPVOTES}, {TITLE}, {DATE}, {FLAIR}
|
||||||
|
# Default: {REDDITOR}_{TITLE}_{POSTID}
|
||||||
|
# BDFR_FILE_SCHEME={REDDITOR}_{TITLE}_{POSTID}
|
||||||
|
|
||||||
|
# Folder naming scheme
|
||||||
|
# Default: {SUBREDDIT}
|
||||||
|
# BDFR_FOLDER_SCHEME={SUBREDDIT}
|
||||||
|
|
||||||
|
# Environment variables for BDFR config location
|
||||||
|
# These tell BDFR where to store its configuration files
|
||||||
|
# APPDATA=/app/data/bdfr-config
|
||||||
|
# XDG_CONFIG_HOME=/app/data/bdfr-config
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# DATABASE CONFIGURATION (Optional)
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# SQLite database file paths (inside container)
|
||||||
|
# These are stored in the mounted ./data directory
|
||||||
|
# DATABASE_URL=sqlite:////app/data/scheduled_tasks.db
|
||||||
|
# SCHEDULER_DATABASE_URL=sqlite:////app/data/scheduler_jobs.db
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# LOGGING CONFIGURATION (Optional)
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
|
||||||
|
# LOG_LEVEL=INFO
|
||||||
|
|
||||||
|
# Log file path (inside container)
|
||||||
|
# LOG_FILE=/app/logs/bdfr-web.log
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# SECURITY CONFIGURATION (Production Only)
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
# Secret key for session management
|
||||||
|
# Generate a secure random string for production
|
||||||
|
# SECRET_KEY=your-secret-key-here
|
||||||
|
|
||||||
|
# Allowed hosts (comma-separated)
|
||||||
|
# ALLOWED_HOSTS=localhost,127.0.0.1,bdfr.example.com
|
||||||
|
|
||||||
|
# Enable CORS (true/false)
|
||||||
|
# ENABLE_CORS=true
|
||||||
|
|
||||||
|
# CORS allowed origins (comma-separated)
|
||||||
|
# CORS_ORIGINS=http://localhost:8000,https://bdfr.example.com
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# NOTES
|
||||||
|
# ============================================================================
|
||||||
|
#
|
||||||
|
# - After modifying this file, restart the Docker container:
|
||||||
|
# docker-compose restart
|
||||||
|
#
|
||||||
|
# - For sensitive values in production, consider using Docker secrets:
|
||||||
|
# https://docs.docker.com/engine/swarm/secrets/
|
||||||
|
#
|
||||||
|
# - Required variables are marked as REQUIRED
|
||||||
|
# - Optional variables will use sensible defaults if not set
|
||||||
|
#
|
||||||
|
# - For more information, see DOCKER.md
|
||||||
@@ -0,0 +1,477 @@
|
|||||||
|
# BDFR Web Interface - Docker Deployment Guide
|
||||||
|
|
||||||
|
This guide covers running the BDFR Web Interface with full BDFR backend support using Docker.
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
### Prerequisites
|
||||||
|
|
||||||
|
- Docker Engine 20.10 or later
|
||||||
|
- Docker Compose 2.0 or later
|
||||||
|
- At least 2GB of free disk space
|
||||||
|
|
||||||
|
### Basic Setup
|
||||||
|
|
||||||
|
1. **Clone the repository** (if you haven't already):
|
||||||
|
```bash
|
||||||
|
git clone <repository-url>
|
||||||
|
cd BDFR_Web
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **Create required directories**:
|
||||||
|
```bash
|
||||||
|
mkdir -p downloads data config
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **Configure environment variables** (optional but recommended):
|
||||||
|
```bash
|
||||||
|
cp web_interface/.env.example .env
|
||||||
|
# Edit .env with your Reddit OAuth credentials
|
||||||
|
```
|
||||||
|
|
||||||
|
4. **Start the container**:
|
||||||
|
```bash
|
||||||
|
docker-compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
5. **Access the web interface**:
|
||||||
|
Open your browser to `http://localhost:8000`
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
### Environment Variables
|
||||||
|
|
||||||
|
Create a `.env` file in the project root with the following variables:
|
||||||
|
|
||||||
|
```env
|
||||||
|
# Reddit OAuth Configuration (Required for authenticated features)
|
||||||
|
BDFR_CLIENT_ID=your_client_id_here
|
||||||
|
BDFR_CLIENT_SECRET=your_client_secret_here
|
||||||
|
BDFR_REDIRECT_URI=http://localhost:8000/auth/callback
|
||||||
|
|
||||||
|
# Server Configuration (Optional)
|
||||||
|
HOST=0.0.0.0
|
||||||
|
PORT=8000
|
||||||
|
DEBUG=false
|
||||||
|
```
|
||||||
|
|
||||||
|
### Getting Reddit OAuth Credentials
|
||||||
|
|
||||||
|
1. Go to https://www.reddit.com/prefs/apps
|
||||||
|
2. Click "Create App" or "Create Another App"
|
||||||
|
3. Fill in the form:
|
||||||
|
- **Name**: BDFR Web Interface (or your choice)
|
||||||
|
- **App type**: Select "web app"
|
||||||
|
- **Description**: (optional)
|
||||||
|
- **About URL**: (optional)
|
||||||
|
- **Redirect URI**: `http://localhost:8000/auth/callback`
|
||||||
|
4. Click "Create app"
|
||||||
|
5. Copy the **client ID** (under the app name) and **client secret**
|
||||||
|
|
||||||
|
### Volume Mounts
|
||||||
|
|
||||||
|
The Docker setup uses two volume mounts:
|
||||||
|
|
||||||
|
| Host Path | Container Path | Purpose |
|
||||||
|
|-----------|----------------|---------|
|
||||||
|
| `./downloads` | `/downloads` | All Reddit downloads are stored here |
|
||||||
|
| `./data` | `/app/data` | SQLite databases, scheduled tasks, and BDFR configuration |
|
||||||
|
|
||||||
|
**Data Directory Structure:**
|
||||||
|
```
|
||||||
|
./data/
|
||||||
|
├── bdfr-config/ # BDFR configuration and OAuth tokens
|
||||||
|
│ ├── config.cfg # BDFR settings
|
||||||
|
│ ├── log_output.txt # BDFR logs
|
||||||
|
│ └── oauth_tokens/ # Reddit authentication tokens
|
||||||
|
├── scheduled_tasks.db # Web interface scheduled tasks
|
||||||
|
└── scheduler_jobs.db # APScheduler job store
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
### Managing the Container
|
||||||
|
|
||||||
|
**Start the container:**
|
||||||
|
```bash
|
||||||
|
docker-compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
**Stop the container:**
|
||||||
|
```bash
|
||||||
|
docker-compose down
|
||||||
|
```
|
||||||
|
|
||||||
|
**View logs:**
|
||||||
|
```bash
|
||||||
|
docker-compose logs -f
|
||||||
|
```
|
||||||
|
|
||||||
|
**Restart the container:**
|
||||||
|
```bash
|
||||||
|
docker-compose restart
|
||||||
|
```
|
||||||
|
|
||||||
|
**Rebuild after code changes:**
|
||||||
|
```bash
|
||||||
|
docker-compose up -d --build
|
||||||
|
```
|
||||||
|
|
||||||
|
### Using the Web Interface
|
||||||
|
|
||||||
|
1. Navigate to `http://localhost:8000`
|
||||||
|
2. Use the web UI to:
|
||||||
|
- Download from subreddits
|
||||||
|
- Download from users
|
||||||
|
- Schedule recurring downloads
|
||||||
|
- Monitor active downloads
|
||||||
|
- View download history
|
||||||
|
|
||||||
|
### Using BDFR CLI Inside Container
|
||||||
|
|
||||||
|
You can also use the BDFR command-line tool directly:
|
||||||
|
|
||||||
|
**Enter the container:**
|
||||||
|
```bash
|
||||||
|
docker exec -it bdfr-web-interface bash
|
||||||
|
```
|
||||||
|
|
||||||
|
**Run BDFR commands:**
|
||||||
|
```bash
|
||||||
|
# Download from a subreddit
|
||||||
|
bdfr download /downloads --subreddit Python -L 10
|
||||||
|
|
||||||
|
# Download from a user
|
||||||
|
bdfr download /downloads --user reddituser --submitted -L 100
|
||||||
|
|
||||||
|
# Archive posts
|
||||||
|
bdfr archive /downloads --subreddit all -L 500
|
||||||
|
|
||||||
|
# Clone (download + archive)
|
||||||
|
bdfr clone /downloads --subreddit EarthPorn -L 50
|
||||||
|
```
|
||||||
|
|
||||||
|
**One-line BDFR commands:**
|
||||||
|
```bash
|
||||||
|
# Download without entering the container
|
||||||
|
docker exec bdfr-web-interface bdfr download /downloads --subreddit Python -L 10
|
||||||
|
|
||||||
|
# View BDFR version
|
||||||
|
docker exec bdfr-web-interface bdfr --version
|
||||||
|
|
||||||
|
# View BDFR help
|
||||||
|
docker exec bdfr-web-interface bdfr download --help
|
||||||
|
```
|
||||||
|
|
||||||
|
## Advanced Configuration
|
||||||
|
|
||||||
|
### Custom Port
|
||||||
|
|
||||||
|
To run on a different port, modify [`docker-compose.yml`](docker-compose.yml:11):
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
ports:
|
||||||
|
- "3000:8000" # Host port 3000, container port 8000
|
||||||
|
```
|
||||||
|
|
||||||
|
Or set the PORT environment variable:
|
||||||
|
```env
|
||||||
|
PORT=3000
|
||||||
|
```
|
||||||
|
|
||||||
|
### Custom Download Directory
|
||||||
|
|
||||||
|
Mount a different host directory for downloads:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
volumes:
|
||||||
|
- /path/to/your/downloads:/downloads
|
||||||
|
- ./data:/app/data
|
||||||
|
```
|
||||||
|
|
||||||
|
### BDFR Configuration File
|
||||||
|
|
||||||
|
BDFR configuration is automatically stored in `./data/bdfr-config/`. You can customize it by creating `./data/bdfr-config/config.cfg`:
|
||||||
|
|
||||||
|
```cfg
|
||||||
|
[DEFAULT]
|
||||||
|
client_id = your_client_id
|
||||||
|
client_secret = your_client_secret
|
||||||
|
scopes = identity, read, history, mysubreddits
|
||||||
|
|
||||||
|
[bdfr]
|
||||||
|
max_wait_time = 120
|
||||||
|
time_format = %Y-%m-%d_%H-%M-%S
|
||||||
|
```
|
||||||
|
|
||||||
|
**Note:** BDFR will automatically create this directory and configuration file on first run.
|
||||||
|
|
||||||
|
### Resource Limits
|
||||||
|
|
||||||
|
Add resource constraints in [`docker-compose.yml`](docker-compose.yml):
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
bdfr-web:
|
||||||
|
# ... other configuration ...
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: '2'
|
||||||
|
memory: 2G
|
||||||
|
reservations:
|
||||||
|
cpus: '1'
|
||||||
|
memory: 512M
|
||||||
|
```
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Container Won't Start
|
||||||
|
|
||||||
|
**Check logs:**
|
||||||
|
```bash
|
||||||
|
docker-compose logs bdfr-web
|
||||||
|
```
|
||||||
|
|
||||||
|
**Verify ports aren't in use:**
|
||||||
|
```bash
|
||||||
|
# Windows
|
||||||
|
netstat -ano | findstr :8000
|
||||||
|
|
||||||
|
# Linux/Mac
|
||||||
|
lsof -i :8000
|
||||||
|
```
|
||||||
|
|
||||||
|
**Rebuild the image:**
|
||||||
|
```bash
|
||||||
|
docker-compose down
|
||||||
|
docker-compose build --no-cache
|
||||||
|
docker-compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
### Permission Issues
|
||||||
|
|
||||||
|
If you encounter permission errors with volumes:
|
||||||
|
|
||||||
|
**Linux/Mac:**
|
||||||
|
```bash
|
||||||
|
sudo chown -R $USER:$USER downloads data config
|
||||||
|
chmod -R 755 downloads data config
|
||||||
|
```
|
||||||
|
|
||||||
|
**Windows:**
|
||||||
|
Ensure Docker Desktop has access to the drive where the project is located (Settings → Resources → File Sharing).
|
||||||
|
|
||||||
|
### Downloads Not Appearing
|
||||||
|
|
||||||
|
1. Check volume mounts are correct:
|
||||||
|
```bash
|
||||||
|
docker inspect bdfr-web-interface | grep -A 10 Mounts
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Verify download directory inside container:
|
||||||
|
```bash
|
||||||
|
docker exec bdfr-web-interface ls -la /downloads
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Check container logs for errors:
|
||||||
|
```bash
|
||||||
|
docker-compose logs -f bdfr-web
|
||||||
|
```
|
||||||
|
|
||||||
|
### Permission Denied Errors
|
||||||
|
|
||||||
|
If you see errors like "Permission denied: '/usr/local/lib/python3.11/site-packages/bdfr/default_config.cfg'":
|
||||||
|
|
||||||
|
1. Verify the BDFR config directory exists and is writable:
|
||||||
|
```bash
|
||||||
|
docker exec bdfr-web-interface ls -la /app/data/bdfr-config
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Check environment variables are set correctly:
|
||||||
|
```bash
|
||||||
|
docker exec bdfr-web-interface env | grep BDFR
|
||||||
|
```
|
||||||
|
|
||||||
|
3. If the directory doesn't exist or has wrong permissions:
|
||||||
|
```bash
|
||||||
|
docker-compose down
|
||||||
|
docker-compose up -d --build
|
||||||
|
```
|
||||||
|
|
||||||
|
### OAuth Authentication Issues
|
||||||
|
|
||||||
|
1. Verify credentials in `.env` or [`docker-compose.yml`](docker-compose.yml)
|
||||||
|
2. Ensure redirect URI matches exactly: `http://localhost:8000/auth/callback`
|
||||||
|
3. Check Reddit app settings at https://www.reddit.com/prefs/apps
|
||||||
|
4. Restart container after changing credentials:
|
||||||
|
```bash
|
||||||
|
docker-compose restart
|
||||||
|
```
|
||||||
|
|
||||||
|
### High Memory Usage
|
||||||
|
|
||||||
|
BDFR can use significant memory when downloading large files or many files simultaneously:
|
||||||
|
|
||||||
|
1. Monitor memory usage:
|
||||||
|
```bash
|
||||||
|
docker stats bdfr-web-interface
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Add memory limits (see Resource Limits above)
|
||||||
|
|
||||||
|
3. Reduce concurrent downloads by limiting the number of simultaneous operations
|
||||||
|
|
||||||
|
### Database Locked Errors
|
||||||
|
|
||||||
|
If you see SQLite database locked errors:
|
||||||
|
|
||||||
|
1. Ensure only one instance is running:
|
||||||
|
```bash
|
||||||
|
docker ps | grep bdfr
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Stop all instances and restart:
|
||||||
|
```bash
|
||||||
|
docker-compose down
|
||||||
|
docker-compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
## Maintenance
|
||||||
|
|
||||||
|
### Backing Up Data
|
||||||
|
|
||||||
|
**Backup downloads and database:**
|
||||||
|
```bash
|
||||||
|
# Create backup directory
|
||||||
|
mkdir -p backups
|
||||||
|
|
||||||
|
# Backup downloads
|
||||||
|
tar -czf backups/downloads-$(date +%Y%m%d).tar.gz downloads/
|
||||||
|
|
||||||
|
# Backup database
|
||||||
|
cp -r data/ backups/data-$(date +%Y%m%d)/
|
||||||
|
```
|
||||||
|
|
||||||
|
**Automated backup script** (Linux/Mac):
|
||||||
|
```bash
|
||||||
|
#!/bin/bash
|
||||||
|
# backup.sh
|
||||||
|
BACKUP_DIR="backups"
|
||||||
|
DATE=$(date +%Y%m%d_%H%M%S)
|
||||||
|
|
||||||
|
mkdir -p "$BACKUP_DIR"
|
||||||
|
tar -czf "$BACKUP_DIR/bdfr-backup-$DATE.tar.gz" downloads/ data/
|
||||||
|
|
||||||
|
# Keep only last 7 days of backups
|
||||||
|
find "$BACKUP_DIR" -name "bdfr-backup-*.tar.gz" -mtime +7 -delete
|
||||||
|
```
|
||||||
|
|
||||||
|
### Updating the Container
|
||||||
|
|
||||||
|
**Pull latest changes:**
|
||||||
|
```bash
|
||||||
|
git pull origin main
|
||||||
|
docker-compose down
|
||||||
|
docker-compose build --no-cache
|
||||||
|
docker-compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
**Update only Docker image:**
|
||||||
|
```bash
|
||||||
|
docker-compose pull
|
||||||
|
docker-compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
### Cleaning Up
|
||||||
|
|
||||||
|
**Remove stopped containers:**
|
||||||
|
```bash
|
||||||
|
docker-compose down
|
||||||
|
```
|
||||||
|
|
||||||
|
**Remove containers and volumes:**
|
||||||
|
```bash
|
||||||
|
docker-compose down -v
|
||||||
|
```
|
||||||
|
|
||||||
|
**Remove images:**
|
||||||
|
```bash
|
||||||
|
docker-compose down --rmi all
|
||||||
|
```
|
||||||
|
|
||||||
|
**Full cleanup (including downloads):**
|
||||||
|
```bash
|
||||||
|
docker-compose down -v --rmi all
|
||||||
|
rm -rf downloads/ data/
|
||||||
|
```
|
||||||
|
|
||||||
|
## Security Considerations
|
||||||
|
|
||||||
|
### Production Deployment
|
||||||
|
|
||||||
|
For production use, consider:
|
||||||
|
|
||||||
|
1. **Use HTTPS**: Set up a reverse proxy (nginx/Traefik) with SSL
|
||||||
|
2. **Secure credentials**: Use Docker secrets or vault for sensitive data
|
||||||
|
3. **Network isolation**: Use custom networks and restrict access
|
||||||
|
4. **Regular updates**: Keep the container and dependencies updated
|
||||||
|
5. **Monitor logs**: Set up log aggregation and monitoring
|
||||||
|
6. **Backup strategy**: Implement automated backups
|
||||||
|
|
||||||
|
### Example Nginx Reverse Proxy
|
||||||
|
|
||||||
|
```nginx
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
server_name bdfr.example.com;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_pass http://localhost:8000;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
# WebSocket support
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection "upgrade";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Performance Optimization
|
||||||
|
|
||||||
|
### Build Performance
|
||||||
|
|
||||||
|
**Use BuildKit for faster builds:**
|
||||||
|
```bash
|
||||||
|
DOCKER_BUILDKIT=1 docker-compose build
|
||||||
|
```
|
||||||
|
|
||||||
|
**Multi-stage build** is already implemented in the [`Dockerfile`](Dockerfile) to minimize image size.
|
||||||
|
|
||||||
|
### Runtime Performance
|
||||||
|
|
||||||
|
1. **Allocate sufficient resources** (see Resource Limits)
|
||||||
|
2. **Use SSD storage** for downloads directory
|
||||||
|
3. **Optimize network** for faster downloads
|
||||||
|
4. **Monitor container health**:
|
||||||
|
```bash
|
||||||
|
docker inspect --format='{{.State.Health.Status}}' bdfr-web-interface
|
||||||
|
```
|
||||||
|
|
||||||
|
## Support
|
||||||
|
|
||||||
|
For issues specific to:
|
||||||
|
- **Docker setup**: Check this guide and Docker logs
|
||||||
|
- **BDFR functionality**: See main [README.md](README.md)
|
||||||
|
- **Web interface**: See [web_interface/README.md](web_interface/README.md)
|
||||||
|
|
||||||
|
## Additional Resources
|
||||||
|
|
||||||
|
- [BDFR Documentation](README.md)
|
||||||
|
- [Docker Documentation](https://docs.docker.com/)
|
||||||
|
- [Docker Compose Documentation](https://docs.docker.com/compose/)
|
||||||
|
- [Reddit API Documentation](https://www.reddit.com/dev/api)
|
||||||
+88
@@ -0,0 +1,88 @@
|
|||||||
|
# Multi-stage Dockerfile for BDFR Web Interface with full BDFR backend support
|
||||||
|
# This image contains both the web interface and the complete BDFR tool
|
||||||
|
|
||||||
|
FROM python:3.11-slim AS builder
|
||||||
|
|
||||||
|
# Set working directory
|
||||||
|
WORKDIR /build
|
||||||
|
|
||||||
|
# Install build dependencies
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
gcc \
|
||||||
|
g++ \
|
||||||
|
git \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Copy BDFR source files and install it
|
||||||
|
COPY pyproject.toml ./
|
||||||
|
COPY bdfr/ ./bdfr/
|
||||||
|
RUN pip install --no-cache-dir .
|
||||||
|
|
||||||
|
# Copy and install web interface requirements
|
||||||
|
COPY web_interface/requirements.txt ./web_requirements.txt
|
||||||
|
RUN pip install --no-cache-dir -r web_requirements.txt
|
||||||
|
|
||||||
|
|
||||||
|
# Final stage - minimal runtime image
|
||||||
|
FROM python:3.11-slim
|
||||||
|
|
||||||
|
# Set labels
|
||||||
|
LABEL maintainer="BDFR Web Interface"
|
||||||
|
LABEL description="Complete BDFR with Web Interface - Download Reddit content via CLI or Web UI"
|
||||||
|
LABEL version="1.0.0"
|
||||||
|
|
||||||
|
# Set working directory
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Install runtime dependencies (ffmpeg for video processing, curl for health checks)
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
ffmpeg \
|
||||||
|
curl \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Copy Python packages from builder (installed globally in /usr/local)
|
||||||
|
COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
|
||||||
|
COPY --from=builder /usr/local/bin /usr/local/bin
|
||||||
|
|
||||||
|
# Copy web interface files
|
||||||
|
COPY web_interface/app/ ./app/
|
||||||
|
COPY web_interface/templates/ ./templates/
|
||||||
|
COPY web_interface/static/ ./static/
|
||||||
|
COPY web_interface/start.py ./start.py
|
||||||
|
|
||||||
|
# Copy entrypoint script
|
||||||
|
COPY docker-entrypoint.sh /usr/local/bin/
|
||||||
|
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
|
||||||
|
|
||||||
|
# Create necessary directories with proper permissions
|
||||||
|
RUN mkdir -p /downloads /app/data /app/logs && \
|
||||||
|
chmod 755 /downloads /app/data /app/logs
|
||||||
|
|
||||||
|
# Create a non-root user for running the application
|
||||||
|
RUN useradd --create-home --shell /bin/bash bdfr && \
|
||||||
|
chown -R bdfr:bdfr /app /downloads
|
||||||
|
|
||||||
|
# Switch to non-root user
|
||||||
|
USER bdfr
|
||||||
|
|
||||||
|
# Set environment variables
|
||||||
|
ENV PYTHONUNBUFFERED=1
|
||||||
|
ENV BDFR_DOWNLOAD_DIR=/downloads
|
||||||
|
ENV BDFR_DATA_DIR=/app/data
|
||||||
|
ENV BDFR_CONFIG_DIR=/app/data/bdfr-config
|
||||||
|
ENV APPDATA=/app/data/bdfr-config
|
||||||
|
ENV XDG_CONFIG_HOME=/app/data/bdfr-config
|
||||||
|
|
||||||
|
# Expose port for web interface
|
||||||
|
EXPOSE 8000
|
||||||
|
|
||||||
|
# Health check - verify web interface is responding
|
||||||
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \
|
||||||
|
CMD curl -f http://localhost:8000/health || exit 1
|
||||||
|
|
||||||
|
# Set entrypoint to handle runtime directory creation
|
||||||
|
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
|
||||||
|
|
||||||
|
# Default command - start web interface
|
||||||
|
# Users can override this to run BDFR CLI commands
|
||||||
|
CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||||
@@ -47,6 +47,31 @@ If on Arch Linux or derivative operating systems such as Manjaro, the BDFR can b
|
|||||||
|
|
||||||
If you want to use the source code or make contributions, refer to [CONTRIBUTING](docs/CONTRIBUTING.md#preparing-the-environment-for-development)
|
If you want to use the source code or make contributions, refer to [CONTRIBUTING](docs/CONTRIBUTING.md#preparing-the-environment-for-development)
|
||||||
|
|
||||||
|
### Docker Deployment
|
||||||
|
|
||||||
|
The BDFR Web Interface provides a complete Docker setup with both the CLI tool and web interface. This is the easiest way to get started:
|
||||||
|
|
||||||
|
**Quick Start (Linux/Mac):**
|
||||||
|
```bash
|
||||||
|
chmod +x docker-quick-start.sh
|
||||||
|
./docker-quick-start.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
**Quick Start (Windows):**
|
||||||
|
```cmd
|
||||||
|
docker-quick-start.bat
|
||||||
|
```
|
||||||
|
|
||||||
|
**Manual Start:**
|
||||||
|
```bash
|
||||||
|
docker-compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
The web interface will be available at `http://localhost:8000`. All downloads are stored in the `./downloads` directory on your host machine.
|
||||||
|
|
||||||
|
For detailed Docker documentation, including configuration, volume management, and troubleshooting, see [DOCKER.md](DOCKER.md).
|
||||||
|
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
The BDFR works by taking submissions from a variety of "sources" from Reddit and then parsing them to download. These sources might be a subreddit, multireddit, a user list, or individual links. These sources are combined and downloaded to disk, according to a naming and organisational scheme defined by the user.
|
The BDFR works by taking submissions from a variety of "sources" from Reddit and then parsing them to download. These sources might be a subreddit, multireddit, a user list, or individual links. These sources are combined and downloaded to disk, according to a naming and organisational scheme defined by the user.
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
bdfr-web:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
container_name: bdfr-web-interface
|
||||||
|
ports:
|
||||||
|
- "8000:8000"
|
||||||
|
volumes:
|
||||||
|
# Downloads directory - all Reddit content goes here
|
||||||
|
- ./downloads:/downloads
|
||||||
|
# Database persistence - SQLite databases, scheduled tasks, and BDFR config
|
||||||
|
- ./data:/app/data
|
||||||
|
environment:
|
||||||
|
# Reddit OAuth Configuration
|
||||||
|
# IMPORTANT: Set these in a .env file or directly here
|
||||||
|
- BDFR_CLIENT_ID=${BDFR_CLIENT_ID:-}
|
||||||
|
- BDFR_CLIENT_SECRET=${BDFR_CLIENT_SECRET:-}
|
||||||
|
- BDFR_REDIRECT_URI=${BDFR_REDIRECT_URI:-http://localhost:8000/auth/callback}
|
||||||
|
|
||||||
|
# Server Configuration
|
||||||
|
- HOST=${HOST:-0.0.0.0}
|
||||||
|
- PORT=${PORT:-8000}
|
||||||
|
- DEBUG=${DEBUG:-false}
|
||||||
|
|
||||||
|
# Download Configuration
|
||||||
|
- BDFR_DOWNLOAD_DIR=/downloads
|
||||||
|
- BDFR_DATA_DIR=/app/data
|
||||||
|
|
||||||
|
# BDFR Configuration - stored in mounted data directory
|
||||||
|
- BDFR_CONFIG_DIR=/app/data/bdfr-config
|
||||||
|
- APPDATA=/app/data/bdfr-config
|
||||||
|
- XDG_CONFIG_HOME=/app/data/bdfr-config
|
||||||
|
|
||||||
|
# Python Configuration
|
||||||
|
- PYTHONUNBUFFERED=1
|
||||||
|
restart: unless-stopped
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 3
|
||||||
|
start_period: 10s
|
||||||
|
networks:
|
||||||
|
- bdfr-network
|
||||||
|
|
||||||
|
networks:
|
||||||
|
bdfr-network:
|
||||||
|
driver: bridge
|
||||||
|
|
||||||
|
# Named volumes configuration (optional)
|
||||||
|
# Uncomment to use named volumes instead of bind mounts
|
||||||
|
# volumes:
|
||||||
|
# downloads:
|
||||||
|
# data:
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Create BDFR config directory if it doesn't exist
|
||||||
|
# This needs to be done at runtime because /app/data is a volume mount
|
||||||
|
mkdir -p /app/data/bdfr-config
|
||||||
|
chmod 755 /app/data/bdfr-config
|
||||||
|
|
||||||
|
# Execute the main command
|
||||||
|
exec "$@"
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
@echo off
|
||||||
|
REM Quick start script for BDFR Web Interface with Docker (Windows)
|
||||||
|
REM This script sets up and starts the Docker environment
|
||||||
|
|
||||||
|
echo ======================================
|
||||||
|
echo BDFR Web Interface - Docker Quick Start
|
||||||
|
echo ======================================
|
||||||
|
echo.
|
||||||
|
|
||||||
|
REM Check if Docker is installed
|
||||||
|
docker --version >nul 2>&1
|
||||||
|
if %errorlevel% neq 0 (
|
||||||
|
echo Error: Docker is not installed
|
||||||
|
echo Please install Docker Desktop from: https://docs.docker.com/desktop/install/windows-install/
|
||||||
|
pause
|
||||||
|
exit /b 1
|
||||||
|
)
|
||||||
|
|
||||||
|
echo Docker is installed
|
||||||
|
echo.
|
||||||
|
|
||||||
|
REM Check if Docker Compose is installed
|
||||||
|
docker-compose --version >nul 2>&1
|
||||||
|
if %errorlevel% neq 0 (
|
||||||
|
docker compose version >nul 2>&1
|
||||||
|
if %errorlevel% neq 0 (
|
||||||
|
echo Error: Docker Compose is not installed
|
||||||
|
echo Please ensure Docker Desktop is properly installed
|
||||||
|
pause
|
||||||
|
exit /b 1
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
echo Docker Compose is installed
|
||||||
|
echo.
|
||||||
|
|
||||||
|
REM Create required directories
|
||||||
|
echo Creating required directories...
|
||||||
|
if not exist "downloads" mkdir downloads
|
||||||
|
if not exist "data" mkdir data
|
||||||
|
if not exist "config" mkdir config
|
||||||
|
if not exist "logs" mkdir logs
|
||||||
|
|
||||||
|
echo Directories created
|
||||||
|
echo.
|
||||||
|
|
||||||
|
REM Check if .env file exists
|
||||||
|
if not exist ".env" (
|
||||||
|
echo Creating .env file from template...
|
||||||
|
if exist ".env.example" (
|
||||||
|
copy .env.example .env >nul
|
||||||
|
echo .env file created
|
||||||
|
echo.
|
||||||
|
echo IMPORTANT: Edit .env file with your Reddit OAuth credentials
|
||||||
|
echo Get credentials from: https://www.reddit.com/prefs/apps
|
||||||
|
echo.
|
||||||
|
pause
|
||||||
|
) else (
|
||||||
|
echo Warning: .env.example not found
|
||||||
|
)
|
||||||
|
) else (
|
||||||
|
echo .env file already exists
|
||||||
|
)
|
||||||
|
echo.
|
||||||
|
|
||||||
|
REM Build and start containers
|
||||||
|
echo Building Docker image...
|
||||||
|
echo This may take a few minutes on first run...
|
||||||
|
echo.
|
||||||
|
|
||||||
|
docker-compose --version >nul 2>&1
|
||||||
|
if %errorlevel% equ 0 (
|
||||||
|
docker-compose build
|
||||||
|
) else (
|
||||||
|
docker compose build
|
||||||
|
)
|
||||||
|
|
||||||
|
if %errorlevel% neq 0 (
|
||||||
|
echo.
|
||||||
|
echo Error: Failed to build Docker image
|
||||||
|
pause
|
||||||
|
exit /b 1
|
||||||
|
)
|
||||||
|
|
||||||
|
echo.
|
||||||
|
echo Build complete!
|
||||||
|
echo.
|
||||||
|
|
||||||
|
echo Starting containers...
|
||||||
|
docker-compose --version >nul 2>&1
|
||||||
|
if %errorlevel% equ 0 (
|
||||||
|
docker-compose up -d
|
||||||
|
) else (
|
||||||
|
docker compose up -d
|
||||||
|
)
|
||||||
|
|
||||||
|
if %errorlevel% neq 0 (
|
||||||
|
echo.
|
||||||
|
echo Error: Failed to start containers
|
||||||
|
pause
|
||||||
|
exit /b 1
|
||||||
|
)
|
||||||
|
|
||||||
|
echo.
|
||||||
|
echo ======================================
|
||||||
|
echo BDFR Web Interface is starting!
|
||||||
|
echo ======================================
|
||||||
|
echo.
|
||||||
|
echo Web Interface: http://localhost:8000
|
||||||
|
echo API Documentation: http://localhost:8000/docs
|
||||||
|
echo.
|
||||||
|
echo Useful commands:
|
||||||
|
echo View logs: docker-compose logs -f
|
||||||
|
echo Stop container: docker-compose down
|
||||||
|
echo Restart: docker-compose restart
|
||||||
|
echo.
|
||||||
|
echo For more information, see DOCKER.md
|
||||||
|
echo.
|
||||||
|
|
||||||
|
REM Wait a moment and check if container is running
|
||||||
|
timeout /t 3 >nul
|
||||||
|
|
||||||
|
docker ps | findstr "bdfr-web-interface" >nul 2>&1
|
||||||
|
if %errorlevel% equ 0 (
|
||||||
|
echo Container is running successfully!
|
||||||
|
) else (
|
||||||
|
echo Warning: Container may not be running. Check logs with:
|
||||||
|
echo docker-compose logs
|
||||||
|
)
|
||||||
|
|
||||||
|
echo.
|
||||||
|
pause
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Quick start script for BDFR Web Interface with Docker
|
||||||
|
# This script sets up and starts the Docker environment
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
echo "======================================"
|
||||||
|
echo "BDFR Web Interface - Docker Quick Start"
|
||||||
|
echo "======================================"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Check if Docker is installed
|
||||||
|
if ! command -v docker &> /dev/null; then
|
||||||
|
echo "❌ Error: Docker is not installed"
|
||||||
|
echo "Please install Docker from: https://docs.docker.com/get-docker/"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if Docker Compose is installed
|
||||||
|
if ! command -v docker-compose &> /dev/null && ! docker compose version &> /dev/null; then
|
||||||
|
echo "❌ Error: Docker Compose is not installed"
|
||||||
|
echo "Please install Docker Compose from: https://docs.docker.com/compose/install/"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "✅ Docker is installed"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Create required directories
|
||||||
|
echo "Creating required directories..."
|
||||||
|
mkdir -p downloads data config logs
|
||||||
|
|
||||||
|
echo "✅ Directories created"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Check if .env file exists
|
||||||
|
if [ ! -f .env ]; then
|
||||||
|
echo "📝 Creating .env file from template..."
|
||||||
|
if [ -f .env.example ]; then
|
||||||
|
cp .env.example .env
|
||||||
|
echo "✅ Created .env file"
|
||||||
|
echo ""
|
||||||
|
echo "⚠️ IMPORTANT: Edit .env file with your Reddit OAuth credentials"
|
||||||
|
echo " Get credentials from: https://www.reddit.com/prefs/apps"
|
||||||
|
echo ""
|
||||||
|
read -p "Press Enter to continue or Ctrl+C to exit and configure .env first..."
|
||||||
|
else
|
||||||
|
echo "⚠️ Warning: .env.example not found"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "✅ .env file already exists"
|
||||||
|
fi
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Build and start containers
|
||||||
|
echo "Building Docker image..."
|
||||||
|
echo "This may take a few minutes on first run..."
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
if command -v docker-compose &> /dev/null; then
|
||||||
|
docker-compose build
|
||||||
|
else
|
||||||
|
docker compose build
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "✅ Build complete!"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
echo "Starting containers..."
|
||||||
|
if command -v docker-compose &> /dev/null; then
|
||||||
|
docker-compose up -d
|
||||||
|
else
|
||||||
|
docker compose up -d
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "======================================"
|
||||||
|
echo "✅ BDFR Web Interface is starting!"
|
||||||
|
echo "======================================"
|
||||||
|
echo ""
|
||||||
|
echo "🌐 Web Interface: http://localhost:8000"
|
||||||
|
echo "📚 API Documentation: http://localhost:8000/docs"
|
||||||
|
echo ""
|
||||||
|
echo "Useful commands:"
|
||||||
|
echo " View logs: docker-compose logs -f"
|
||||||
|
echo " Stop container: docker-compose down"
|
||||||
|
echo " Restart: docker-compose restart"
|
||||||
|
echo ""
|
||||||
|
echo "For more information, see DOCKER.md"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Wait a moment and check if container is running
|
||||||
|
sleep 3
|
||||||
|
|
||||||
|
if command -v docker-compose &> /dev/null; then
|
||||||
|
if docker-compose ps | grep -q "Up"; then
|
||||||
|
echo "✅ Container is running successfully!"
|
||||||
|
else
|
||||||
|
echo "⚠️ Container may not be running. Check logs with:"
|
||||||
|
echo " docker-compose logs"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
if docker compose ps | grep -q "Up"; then
|
||||||
|
echo "✅ Container is running successfully!"
|
||||||
|
else
|
||||||
|
echo "⚠️ Container may not be running. Check logs with:"
|
||||||
|
echo " docker compose logs"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
# BDFR Web Interface Configuration
|
|
||||||
# Copy this file to .env and update the values as needed
|
|
||||||
|
|
||||||
# Reddit OAuth Configuration
|
|
||||||
# You MUST set this to match your Reddit OAuth app settings
|
|
||||||
# Go to https://www.reddit.com/prefs/apps, create/edit your app, and use the exact redirect URI
|
|
||||||
BDFR_REDIRECT_URI=http://localhost:8000/auth/callback
|
|
||||||
|
|
||||||
# OAuth Credentials (from your Reddit OAuth app)
|
|
||||||
# Get these from: https://www.reddit.com/prefs/apps
|
|
||||||
BDFR_CLIENT_ID=your_client_id_here
|
|
||||||
BDFR_CLIENT_SECRET=your_client_secret_here
|
|
||||||
|
|
||||||
# Server Configuration (optional)
|
|
||||||
# HOST=0.0.0.0
|
|
||||||
# PORT=8000
|
|
||||||
# DEBUG=true
|
|
||||||
@@ -0,0 +1,319 @@
|
|||||||
|
# Scheduled Downloads Feature
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
The BDFR Web Interface now supports scheduled downloads, allowing users to configure downloads that run automatically on a daily basis. This feature is designed to work seamlessly in Docker containers and ensures sequential execution to prevent system overload.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
### 1. **Daily Scheduling**
|
||||||
|
- Tasks run once per day at a user-specified time
|
||||||
|
- Time is specified in the user's local timezone and automatically converted to UTC for container execution
|
||||||
|
- Automatically sets `time_filter="day"` to only download content from the last 24 hours
|
||||||
|
- Always enables `no_dupes=True` to avoid re-downloading existing content
|
||||||
|
|
||||||
|
### 2. **Sequential Execution**
|
||||||
|
- Tasks are processed one at a time through a queue system
|
||||||
|
- Manual "Run Now" tasks have priority over scheduled tasks
|
||||||
|
- Queue status is displayed in real-time
|
||||||
|
|
||||||
|
### 3. **Persistent Storage**
|
||||||
|
- SQLite database stores task configurations
|
||||||
|
- APScheduler job store ensures tasks persist across container restarts
|
||||||
|
- Execution history tracked for each task
|
||||||
|
|
||||||
|
### 4. **Full Task Management**
|
||||||
|
- Create, enable/disable, and delete scheduled tasks
|
||||||
|
- Run tasks manually on-demand
|
||||||
|
- View last run and next scheduled run times
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
### Backend Components
|
||||||
|
|
||||||
|
#### 1. **Database Models** (`web_interface/app/models.py`)
|
||||||
|
- `ScheduledTask`: Stores task configuration
|
||||||
|
- Fields: name, source (subreddit/user), schedule, timezone, etc.
|
||||||
|
- Automatically sets time_filter="day" and no_dupes=True
|
||||||
|
- `TaskExecutionHistory`: Tracks each execution
|
||||||
|
- Fields: task_id, status, items downloaded, errors, etc.
|
||||||
|
|
||||||
|
#### 2. **Task Queue** (`web_interface/app/task_queue.py`)
|
||||||
|
- `TaskQueue` class manages sequential execution
|
||||||
|
- Priority queue: 0=scheduled, 1=manual
|
||||||
|
- Blocks until each download completes before starting the next
|
||||||
|
- Thread-safe using asyncio
|
||||||
|
|
||||||
|
#### 3. **Scheduler Service** (`web_interface/app/scheduler.py`)
|
||||||
|
- APScheduler with SQLAlchemy job store for persistence
|
||||||
|
- Functions:
|
||||||
|
- `schedule_task()`: Creates cron job
|
||||||
|
- `queue_scheduled_task()`: Adds task to queue (called by scheduler)
|
||||||
|
- `execute_scheduled_task()`: Executes download and waits for completion
|
||||||
|
- `wait_for_download_completion()`: Polls every 5 seconds until done
|
||||||
|
|
||||||
|
#### 4. **API Endpoints** (`web_interface/app/scheduled_tasks.py`)
|
||||||
|
```
|
||||||
|
POST /api/scheduled-tasks - Create task
|
||||||
|
GET /api/scheduled-tasks - List all tasks
|
||||||
|
GET /api/scheduled-tasks/{id} - Get specific task
|
||||||
|
PUT /api/scheduled-tasks/{id} - Update task
|
||||||
|
DELETE /api/scheduled-tasks/{id} - Delete task
|
||||||
|
POST /api/scheduled-tasks/{id}/toggle - Enable/disable
|
||||||
|
POST /api/scheduled-tasks/{id}/run-now - Queue immediately
|
||||||
|
GET /api/scheduled-tasks/{id}/history - Execution history
|
||||||
|
GET /api/scheduled-tasks/queue/status - Queue status
|
||||||
|
```
|
||||||
|
|
||||||
|
### Frontend Components
|
||||||
|
|
||||||
|
#### 1. **HTML** (`web_interface/templates/index.html`)
|
||||||
|
- "Run Daily" checkbox in Advanced Options
|
||||||
|
- Schedule configuration fields (task name, run time)
|
||||||
|
- Scheduled Downloads section with task cards
|
||||||
|
- Queue status badge
|
||||||
|
|
||||||
|
#### 2. **JavaScript** (`web_interface/static/js/app.js`)
|
||||||
|
- `loadScheduledTasks()`: Fetches and renders tasks
|
||||||
|
- `createScheduledTask()`: Creates new scheduled task
|
||||||
|
- `toggleTask()`, `deleteTask()`, `runTaskNow()`: Task management
|
||||||
|
- `updateQueueStatus()`: Polls queue every 10 seconds
|
||||||
|
- Auto-detects browser timezone via `Intl.DateTimeFormat()`
|
||||||
|
|
||||||
|
#### 3. **CSS** (`web_interface/static/css/style.css`)
|
||||||
|
- Task card styling with hover effects
|
||||||
|
- Status badges (enabled/disabled)
|
||||||
|
- Schedule options panel
|
||||||
|
- Queue status badge
|
||||||
|
|
||||||
|
## Usage Guide
|
||||||
|
|
||||||
|
### Creating a Scheduled Download
|
||||||
|
|
||||||
|
1. **Configure Download Settings**
|
||||||
|
- Select download mode (Download/Archive/Clone)
|
||||||
|
- Choose source type (Subreddit/User)
|
||||||
|
- Enter source name
|
||||||
|
- Set limit, sort, and other options
|
||||||
|
|
||||||
|
2. **Enable Scheduling**
|
||||||
|
- Check "Run Daily" in Advanced Options
|
||||||
|
- Enter a task name (e.g., "Daily Python Posts")
|
||||||
|
- Select run time (24-hour format, in your local timezone)
|
||||||
|
|
||||||
|
3. **Submit**
|
||||||
|
- Click "Start Download" to create the scheduled task
|
||||||
|
- Task appears in the Scheduled Downloads section
|
||||||
|
- First run scheduled for the specified time
|
||||||
|
|
||||||
|
### Managing Scheduled Tasks
|
||||||
|
|
||||||
|
Each task card shows:
|
||||||
|
- Task name and source
|
||||||
|
- Download mode
|
||||||
|
- Schedule (daily at X time)
|
||||||
|
- Last run and next run times
|
||||||
|
- Status (Enabled/Disabled)
|
||||||
|
|
||||||
|
**Actions:**
|
||||||
|
- **Disable/Enable**: Toggle task on/off without deleting
|
||||||
|
- **Run Now**: Add task to queue immediately (higher priority)
|
||||||
|
- **Delete**: Remove task permanently
|
||||||
|
|
||||||
|
### Queue System
|
||||||
|
|
||||||
|
- Queue status badge shows number of tasks waiting
|
||||||
|
- Tasks execute one at a time to prevent overload
|
||||||
|
- Manual "Run Now" tasks have priority over scheduled tasks
|
||||||
|
- Download progress appears in Progress section
|
||||||
|
|
||||||
|
## Docker Deployment
|
||||||
|
|
||||||
|
### Volume Mounts Required
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
volumes:
|
||||||
|
- ./downloads:/downloads # Downloaded files
|
||||||
|
- ./data:/app/data # Database and job store
|
||||||
|
```
|
||||||
|
|
||||||
|
### Database Location
|
||||||
|
- SQLite: `/app/data/scheduled_tasks.db`
|
||||||
|
- APScheduler job store: Same database
|
||||||
|
|
||||||
|
### Timezone Handling
|
||||||
|
- User specifies time in their local timezone
|
||||||
|
- Frontend auto-detects timezone via JavaScript
|
||||||
|
- Backend converts to UTC for container execution
|
||||||
|
- Cron jobs run at correct local time regardless of container timezone
|
||||||
|
|
||||||
|
## Technical Details
|
||||||
|
|
||||||
|
### Sequential Execution Flow
|
||||||
|
|
||||||
|
1. APScheduler triggers at scheduled time
|
||||||
|
2. Scheduler calls `queue_scheduled_task(task_id)`
|
||||||
|
3. Task added to queue with priority 0
|
||||||
|
4. Queue worker picks up task
|
||||||
|
5. `execute_scheduled_task()` called
|
||||||
|
6. Downloads via existing BDFR API
|
||||||
|
7. `wait_for_download_completion()` polls every 5s
|
||||||
|
8. Once complete, queue processes next task
|
||||||
|
9. Execution history recorded
|
||||||
|
|
||||||
|
### Time Filter Logic
|
||||||
|
|
||||||
|
For scheduled tasks:
|
||||||
|
- `time_filter` is automatically set to "day"
|
||||||
|
- This filters Reddit API to only return posts from last 24 hours
|
||||||
|
- Combined with daily scheduling, ensures only new content downloaded
|
||||||
|
- Prevents re-downloading old content
|
||||||
|
|
||||||
|
### Duplicate Prevention
|
||||||
|
|
||||||
|
For scheduled tasks:
|
||||||
|
- `no_dupes` is automatically enabled
|
||||||
|
- Uses existing BDFR duplicate detection
|
||||||
|
- Checks file hashes or URLs (depending on simple_check setting)
|
||||||
|
- Skips files that already exist
|
||||||
|
|
||||||
|
## Testing Checklist
|
||||||
|
|
||||||
|
### Basic Functionality
|
||||||
|
- [ ] Create scheduled task via "Run Daily" checkbox
|
||||||
|
- [ ] Task appears in Scheduled Downloads section
|
||||||
|
- [ ] Task name, source, and schedule displayed correctly
|
||||||
|
- [ ] Enable/disable toggle works
|
||||||
|
- [ ] Delete removes task
|
||||||
|
|
||||||
|
### Execution
|
||||||
|
- [ ] Manual "Run Now" triggers download immediately
|
||||||
|
- [ ] Download progress appears in Progress section
|
||||||
|
- [ ] Task completes successfully
|
||||||
|
- [ ] Execution history recorded
|
||||||
|
|
||||||
|
### Sequential Processing
|
||||||
|
- [ ] Queue multiple tasks via "Run Now"
|
||||||
|
- [ ] Tasks execute one at a time (not concurrent)
|
||||||
|
- [ ] Queue badge shows correct count
|
||||||
|
- [ ] Manual tasks execute before scheduled tasks
|
||||||
|
|
||||||
|
### Persistence
|
||||||
|
- [ ] Restart container/server
|
||||||
|
- [ ] Tasks still present after restart
|
||||||
|
- [ ] Scheduled jobs still execute at correct time
|
||||||
|
- [ ] Execution history preserved
|
||||||
|
|
||||||
|
### Timezone Handling
|
||||||
|
- [ ] Create task with different timezone
|
||||||
|
- [ ] Task runs at correct local time
|
||||||
|
- [ ] Next run time displays in user's timezone
|
||||||
|
|
||||||
|
### Edge Cases
|
||||||
|
- [ ] Create task with invalid source name
|
||||||
|
- [ ] Disable task, verify it doesn't run
|
||||||
|
- [ ] Enable disabled task
|
||||||
|
- [ ] Delete task while it's running
|
||||||
|
- [ ] Run same task multiple times quickly
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Tasks Not Running
|
||||||
|
1. Check container logs for scheduler errors
|
||||||
|
2. Verify `/app/data` volume is mounted
|
||||||
|
3. Check database file permissions
|
||||||
|
4. Verify APScheduler is running (`scheduler.running()`)
|
||||||
|
|
||||||
|
### Queue Stuck
|
||||||
|
1. Check task_queue status in logs
|
||||||
|
2. Verify WebSocket connection for progress updates
|
||||||
|
3. Restart container to reset queue
|
||||||
|
|
||||||
|
### Timezone Issues
|
||||||
|
1. Verify browser timezone detection in DevTools
|
||||||
|
2. Check conversion in scheduler logs
|
||||||
|
3. Ensure container has correct UTC time
|
||||||
|
|
||||||
|
### Database Issues
|
||||||
|
1. Check `/app/data/scheduled_tasks.db` exists
|
||||||
|
2. Verify write permissions
|
||||||
|
3. Use SQLite browser to inspect tables
|
||||||
|
4. Check for migration errors in logs
|
||||||
|
|
||||||
|
## Future Enhancements
|
||||||
|
|
||||||
|
Potential improvements:
|
||||||
|
- [ ] Weekly scheduling option
|
||||||
|
- [ ] Custom time filters (last 3 days, last week, etc.)
|
||||||
|
- [ ] Email notifications on completion/failure
|
||||||
|
- [ ] Retry logic for failed tasks
|
||||||
|
- [ ] Task templates for quick setup
|
||||||
|
- [ ] Bulk operations (enable/disable multiple tasks)
|
||||||
|
- [ ] Advanced schedule expressions (cron syntax)
|
||||||
|
- [ ] Export/import task configurations
|
||||||
|
- [ ] Task execution statistics and charts
|
||||||
|
- [ ] Pause/resume queue
|
||||||
|
|
||||||
|
## API Examples
|
||||||
|
|
||||||
|
### Create Task
|
||||||
|
```bash
|
||||||
|
curl -X POST http://localhost:8000/api/scheduled-tasks \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{
|
||||||
|
"name": "Daily Python Posts",
|
||||||
|
"source_type": "subreddit",
|
||||||
|
"source_name": "python",
|
||||||
|
"download_mode": "download",
|
||||||
|
"limit": 50,
|
||||||
|
"sort": "hot",
|
||||||
|
"run_time": "02:00",
|
||||||
|
"timezone": "Pacific/Auckland",
|
||||||
|
"enabled": true
|
||||||
|
}'
|
||||||
|
```
|
||||||
|
|
||||||
|
### List Tasks
|
||||||
|
```bash
|
||||||
|
curl http://localhost:8000/api/scheduled-tasks
|
||||||
|
```
|
||||||
|
|
||||||
|
### Toggle Task
|
||||||
|
```bash
|
||||||
|
curl -X POST http://localhost:8000/api/scheduled-tasks/1/toggle
|
||||||
|
```
|
||||||
|
|
||||||
|
### Run Task Now
|
||||||
|
```bash
|
||||||
|
curl -X POST http://localhost:8000/api/scheduled-tasks/1/run-now
|
||||||
|
```
|
||||||
|
|
||||||
|
### Get Queue Status
|
||||||
|
```bash
|
||||||
|
curl http://localhost:8000/api/scheduled-tasks/queue/status
|
||||||
|
```
|
||||||
|
|
||||||
|
## Files Modified/Created
|
||||||
|
|
||||||
|
### Created:
|
||||||
|
- `web_interface/app/database.py` - Database configuration
|
||||||
|
- `web_interface/app/models.py` - ORM models
|
||||||
|
- `web_interface/app/task_queue.py` - Queue manager
|
||||||
|
- `web_interface/app/scheduler.py` - Scheduler service
|
||||||
|
- `web_interface/app/scheduled_tasks.py` - API endpoints
|
||||||
|
- `web_interface/SCHEDULED_DOWNLOADS.md` - This file
|
||||||
|
|
||||||
|
### Modified:
|
||||||
|
- `web_interface/requirements.txt` - Added dependencies
|
||||||
|
- `web_interface/app/main.py` - Integrated scheduler
|
||||||
|
- `web_interface/templates/index.html` - Added UI elements
|
||||||
|
- `web_interface/static/js/app.js` - Added JavaScript functions
|
||||||
|
- `web_interface/static/css/style.css` - Added styles
|
||||||
|
|
||||||
|
## Dependencies Added
|
||||||
|
|
||||||
|
```
|
||||||
|
sqlalchemy>=2.0.0
|
||||||
|
alembic>=1.12.0
|
||||||
|
apscheduler>=3.10.0
|
||||||
|
pytz>=2023.3
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
"""
|
||||||
|
Database configuration for scheduled downloads.
|
||||||
|
Uses SQLite with SQLAlchemy ORM for persistent storage.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine, event
|
||||||
|
from sqlalchemy.ext.declarative import declarative_base
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
from pathlib import Path
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Database file location - uses /app/data in Docker container
|
||||||
|
# This directory should be mounted as a volume for persistence
|
||||||
|
DATA_DIR = Path(__file__).parent.parent / "data"
|
||||||
|
DB_PATH = DATA_DIR / "scheduled_tasks.db"
|
||||||
|
|
||||||
|
# Ensure data directory exists (will be mounted volume in Docker)
|
||||||
|
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Create engine with proper settings for SQLite in Docker
|
||||||
|
engine = create_engine(
|
||||||
|
f"sqlite:///{DB_PATH}",
|
||||||
|
echo=False,
|
||||||
|
connect_args={
|
||||||
|
"check_same_thread": False, # Allow multi-threaded access
|
||||||
|
"timeout": 30 # Longer timeout for container I/O
|
||||||
|
},
|
||||||
|
pool_pre_ping=True, # Verify connections before using
|
||||||
|
)
|
||||||
|
|
||||||
|
# Enable foreign keys for SQLite
|
||||||
|
@event.listens_for(engine, "connect")
|
||||||
|
def set_sqlite_pragma(dbapi_conn, connection_record):
|
||||||
|
cursor = dbapi_conn.cursor()
|
||||||
|
cursor.execute("PRAGMA foreign_keys=ON")
|
||||||
|
cursor.close()
|
||||||
|
|
||||||
|
# Create session factory
|
||||||
|
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||||
|
|
||||||
|
# Base class for models
|
||||||
|
Base = declarative_base()
|
||||||
|
|
||||||
|
|
||||||
|
def init_database():
|
||||||
|
"""Initialize database - called on container startup"""
|
||||||
|
try:
|
||||||
|
Base.metadata.create_all(bind=engine)
|
||||||
|
logger.info(f"Database initialized at {DB_PATH}")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to initialize database: {e}")
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def get_db():
|
||||||
|
"""
|
||||||
|
Dependency for FastAPI to get database session.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
@app.get("/endpoint")
|
||||||
|
def endpoint(db: Session = Depends(get_db)):
|
||||||
|
...
|
||||||
|
"""
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
yield db
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
@@ -28,6 +28,11 @@ logger = logging.getLogger(__name__)
|
|||||||
# Import authentication module
|
# Import authentication module
|
||||||
from .auth import init_oauth_manager, get_oauth_manager
|
from .auth import init_oauth_manager, get_oauth_manager
|
||||||
|
|
||||||
|
# Import scheduled tasks modules
|
||||||
|
from .database import init_database
|
||||||
|
from .scheduler import start_scheduler, stop_scheduler
|
||||||
|
from .scheduled_tasks import router as scheduled_tasks_router
|
||||||
|
|
||||||
# Import BDFR API layer
|
# Import BDFR API layer
|
||||||
import sys
|
import sys
|
||||||
import os
|
import os
|
||||||
@@ -72,6 +77,45 @@ except ImportError as e:
|
|||||||
|
|
||||||
app = FastAPI(title="BDFR Web Interface", version="1.0.0")
|
app = FastAPI(title="BDFR Web Interface", version="1.0.0")
|
||||||
|
|
||||||
|
# Application lifecycle events
|
||||||
|
@app.on_event("startup")
|
||||||
|
async def startup_event():
|
||||||
|
"""Initialize services on application startup"""
|
||||||
|
try:
|
||||||
|
logger.info("Starting up BDFR Web Interface...")
|
||||||
|
|
||||||
|
# Initialize database
|
||||||
|
init_database()
|
||||||
|
logger.info("Database initialized")
|
||||||
|
|
||||||
|
# Start scheduler
|
||||||
|
start_scheduler()
|
||||||
|
logger.info("Scheduler started")
|
||||||
|
|
||||||
|
logger.info("Startup complete!")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Startup error: {e}", exc_info=True)
|
||||||
|
raise
|
||||||
|
|
||||||
|
@app.on_event("shutdown")
|
||||||
|
async def shutdown_event():
|
||||||
|
"""Cleanup on application shutdown"""
|
||||||
|
try:
|
||||||
|
logger.info("Shutting down BDFR Web Interface...")
|
||||||
|
|
||||||
|
# Stop scheduler
|
||||||
|
stop_scheduler()
|
||||||
|
logger.info("Scheduler stopped")
|
||||||
|
|
||||||
|
# Stop task queue
|
||||||
|
from .task_queue import task_queue
|
||||||
|
await task_queue.stop()
|
||||||
|
logger.info("Task queue stopped")
|
||||||
|
|
||||||
|
logger.info("Shutdown complete!")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Shutdown error: {e}", exc_info=True)
|
||||||
|
|
||||||
# Initialize OAuth2 manager
|
# Initialize OAuth2 manager
|
||||||
def init_oauth():
|
def init_oauth():
|
||||||
"""Initialize OAuth2 manager with credentials from environment or BDFR config"""
|
"""Initialize OAuth2 manager with credentials from environment or BDFR config"""
|
||||||
@@ -131,6 +175,9 @@ os.makedirs(template_dir, exist_ok=True)
|
|||||||
app.mount("/static", StaticFiles(directory=static_dir), name="static")
|
app.mount("/static", StaticFiles(directory=static_dir), name="static")
|
||||||
templates = Jinja2Templates(directory=template_dir)
|
templates = Jinja2Templates(directory=template_dir)
|
||||||
|
|
||||||
|
# Include scheduled tasks router
|
||||||
|
app.include_router(scheduled_tasks_router)
|
||||||
|
|
||||||
# WebSocket connection manager
|
# WebSocket connection manager
|
||||||
class ConnectionManager:
|
class ConnectionManager:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
|||||||
@@ -0,0 +1,130 @@
|
|||||||
|
"""
|
||||||
|
Database models for scheduled downloads.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from sqlalchemy import Column, String, Integer, Boolean, DateTime, Time, ForeignKey, Text
|
||||||
|
from sqlalchemy.orm import relationship
|
||||||
|
from datetime import datetime, time as time_type
|
||||||
|
import uuid
|
||||||
|
import pytz
|
||||||
|
|
||||||
|
from .database import Base
|
||||||
|
|
||||||
|
|
||||||
|
def generate_uuid():
|
||||||
|
"""Generate a UUID string"""
|
||||||
|
return str(uuid.uuid4())
|
||||||
|
|
||||||
|
|
||||||
|
class ScheduledTask(Base):
|
||||||
|
"""
|
||||||
|
Represents a scheduled download task.
|
||||||
|
Tasks run at a specified time each day.
|
||||||
|
"""
|
||||||
|
__tablename__ = "scheduled_tasks"
|
||||||
|
|
||||||
|
# Primary key
|
||||||
|
id = Column(String(36), primary_key=True, default=generate_uuid)
|
||||||
|
|
||||||
|
# Task identification
|
||||||
|
name = Column(String(255), nullable=False, index=True)
|
||||||
|
enabled = Column(Boolean, default=True, nullable=False, index=True)
|
||||||
|
|
||||||
|
# Download configuration
|
||||||
|
source_type = Column(String(20), nullable=False) # "subreddit" or "user"
|
||||||
|
source_name = Column(String(255), nullable=False, index=True)
|
||||||
|
download_mode = Column(String(20), nullable=False) # "download", "archive", "clone"
|
||||||
|
|
||||||
|
# Filter options
|
||||||
|
limit = Column(Integer, default=25, nullable=False)
|
||||||
|
sort = Column(String(20), default="hot", nullable=False)
|
||||||
|
time_filter = Column(String(20), default="day", nullable=False) # Always "day" for daily tasks
|
||||||
|
min_score = Column(Integer, nullable=True)
|
||||||
|
no_dupes = Column(Boolean, default=True, nullable=False) # Always true for scheduled
|
||||||
|
simple_check = Column(Boolean, default=False, nullable=False)
|
||||||
|
|
||||||
|
# Scheduling
|
||||||
|
schedule_frequency = Column(String(20), default="daily", nullable=False)
|
||||||
|
run_time = Column(Time, nullable=False) # Time of day to run
|
||||||
|
timezone = Column(String(50), default="UTC", nullable=False)
|
||||||
|
|
||||||
|
# Metadata
|
||||||
|
created_at = Column(DateTime, default=lambda: datetime.now(pytz.UTC), nullable=False)
|
||||||
|
updated_at = Column(DateTime, default=lambda: datetime.now(pytz.UTC), onupdate=lambda: datetime.now(pytz.UTC), nullable=False)
|
||||||
|
last_run_at = Column(DateTime, nullable=True, index=True)
|
||||||
|
next_run_at = Column(DateTime, nullable=True, index=True)
|
||||||
|
|
||||||
|
# Authentication
|
||||||
|
auth_state = Column(String(255), nullable=True)
|
||||||
|
|
||||||
|
# Relationships
|
||||||
|
executions = relationship("TaskExecutionHistory", back_populates="task", cascade="all, delete-orphan")
|
||||||
|
|
||||||
|
def to_dict(self):
|
||||||
|
"""Convert to dictionary for JSON serialization"""
|
||||||
|
return {
|
||||||
|
"id": self.id,
|
||||||
|
"name": self.name,
|
||||||
|
"enabled": self.enabled,
|
||||||
|
"source_type": self.source_type,
|
||||||
|
"source_name": self.source_name,
|
||||||
|
"download_mode": self.download_mode,
|
||||||
|
"limit": self.limit,
|
||||||
|
"sort": self.sort,
|
||||||
|
"time_filter": self.time_filter,
|
||||||
|
"min_score": self.min_score,
|
||||||
|
"no_dupes": self.no_dupes,
|
||||||
|
"simple_check": self.simple_check,
|
||||||
|
"schedule_frequency": self.schedule_frequency,
|
||||||
|
"run_time": self.run_time.isoformat() if self.run_time else None,
|
||||||
|
"timezone": self.timezone,
|
||||||
|
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||||
|
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
|
||||||
|
"last_run_at": self.last_run_at.isoformat() if self.last_run_at else None,
|
||||||
|
"next_run_at": self.next_run_at.isoformat() if self.next_run_at else None,
|
||||||
|
"auth_state": self.auth_state
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class TaskExecutionHistory(Base):
|
||||||
|
"""
|
||||||
|
Records each execution of a scheduled task.
|
||||||
|
Tracks success/failure and metrics.
|
||||||
|
"""
|
||||||
|
__tablename__ = "task_execution_history"
|
||||||
|
|
||||||
|
# Primary key
|
||||||
|
id = Column(String(36), primary_key=True, default=generate_uuid)
|
||||||
|
|
||||||
|
# Foreign key to task
|
||||||
|
task_id = Column(String(36), ForeignKey("scheduled_tasks.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||||
|
|
||||||
|
# Execution details
|
||||||
|
started_at = Column(DateTime, default=lambda: datetime.now(pytz.UTC), nullable=False, index=True)
|
||||||
|
completed_at = Column(DateTime, nullable=True)
|
||||||
|
status = Column(String(20), default="queued", nullable=False, index=True) # queued, running, success, failed, skipped
|
||||||
|
|
||||||
|
# Results
|
||||||
|
items_found = Column(Integer, default=0, nullable=False)
|
||||||
|
items_downloaded = Column(Integer, default=0, nullable=False)
|
||||||
|
error_message = Column(Text, nullable=True)
|
||||||
|
|
||||||
|
# Link to download
|
||||||
|
download_id = Column(String(255), nullable=True, index=True)
|
||||||
|
|
||||||
|
# Relationships
|
||||||
|
task = relationship("ScheduledTask", back_populates="executions")
|
||||||
|
|
||||||
|
def to_dict(self):
|
||||||
|
"""Convert to dictionary for JSON serialization"""
|
||||||
|
return {
|
||||||
|
"id": self.id,
|
||||||
|
"task_id": self.task_id,
|
||||||
|
"started_at": self.started_at.isoformat() if self.started_at else None,
|
||||||
|
"completed_at": self.completed_at.isoformat() if self.completed_at else None,
|
||||||
|
"status": self.status,
|
||||||
|
"items_found": self.items_found,
|
||||||
|
"items_downloaded": self.items_downloaded,
|
||||||
|
"error_message": self.error_message,
|
||||||
|
"download_id": self.download_id
|
||||||
|
}
|
||||||
@@ -0,0 +1,474 @@
|
|||||||
|
"""
|
||||||
|
API endpoints and business logic for scheduled tasks management.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, status
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
from typing import List, Optional
|
||||||
|
from datetime import datetime, time as time_type
|
||||||
|
import pytz
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from .database import get_db
|
||||||
|
from .models import ScheduledTask, TaskExecutionHistory
|
||||||
|
from .scheduler import schedule_task, unschedule_task, calculate_next_run
|
||||||
|
from .task_queue import task_queue
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Create router
|
||||||
|
router = APIRouter(prefix="/api/scheduled-tasks", tags=["scheduled-tasks"])
|
||||||
|
|
||||||
|
|
||||||
|
# Pydantic models for API
|
||||||
|
class ScheduledTaskCreate(BaseModel):
|
||||||
|
name: str = Field(..., min_length=1, max_length=255)
|
||||||
|
source_type: str = Field(..., pattern="^(subreddit|user)$")
|
||||||
|
source_name: str = Field(..., min_length=1, max_length=255)
|
||||||
|
download_mode: str = Field(..., pattern="^(download|archive|clone)$")
|
||||||
|
limit: int = Field(default=25, ge=1, le=1000)
|
||||||
|
sort: str = Field(default="hot")
|
||||||
|
min_score: Optional[int] = None
|
||||||
|
simple_check: bool = False
|
||||||
|
run_time: str = Field(..., pattern="^([01]?[0-9]|2[0-3]):[0-5][0-9]$") # HH:MM format
|
||||||
|
timezone: str = Field(default="UTC")
|
||||||
|
auth_state: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class ScheduledTaskUpdate(BaseModel):
|
||||||
|
name: Optional[str] = Field(None, min_length=1, max_length=255)
|
||||||
|
limit: Optional[int] = Field(None, ge=1, le=1000)
|
||||||
|
sort: Optional[str] = None
|
||||||
|
min_score: Optional[int] = None
|
||||||
|
simple_check: Optional[bool] = None
|
||||||
|
run_time: Optional[str] = Field(None, pattern="^([01]?[0-9]|2[0-3]):[0-5][0-9]$")
|
||||||
|
timezone: Optional[str] = None
|
||||||
|
enabled: Optional[bool] = None
|
||||||
|
|
||||||
|
|
||||||
|
class ScheduledTaskResponse(BaseModel):
|
||||||
|
id: str
|
||||||
|
name: str
|
||||||
|
enabled: bool
|
||||||
|
source_type: str
|
||||||
|
source_name: str
|
||||||
|
download_mode: str
|
||||||
|
limit: int
|
||||||
|
sort: str
|
||||||
|
time_filter: str
|
||||||
|
min_score: Optional[int]
|
||||||
|
no_dupes: bool
|
||||||
|
simple_check: bool
|
||||||
|
schedule_frequency: str
|
||||||
|
run_time: str
|
||||||
|
timezone: str
|
||||||
|
created_at: str
|
||||||
|
updated_at: str
|
||||||
|
last_run_at: Optional[str]
|
||||||
|
next_run_at: Optional[str]
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
from_attributes = True
|
||||||
|
|
||||||
|
|
||||||
|
class TaskExecutionResponse(BaseModel):
|
||||||
|
id: str
|
||||||
|
task_id: str
|
||||||
|
started_at: str
|
||||||
|
completed_at: Optional[str]
|
||||||
|
status: str
|
||||||
|
items_found: int
|
||||||
|
items_downloaded: int
|
||||||
|
error_message: Optional[str]
|
||||||
|
download_id: Optional[str]
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
from_attributes = True
|
||||||
|
|
||||||
|
|
||||||
|
# API Endpoints
|
||||||
|
|
||||||
|
@router.post("", response_model=ScheduledTaskResponse, status_code=status.HTTP_201_CREATED)
|
||||||
|
async def create_scheduled_task(task_data: ScheduledTaskCreate, db: Session = Depends(get_db)):
|
||||||
|
"""
|
||||||
|
Create a new scheduled download task.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Validate timezone
|
||||||
|
try:
|
||||||
|
pytz.timezone(task_data.timezone)
|
||||||
|
except pytz.exceptions.UnknownTimeZoneError:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail=f"Invalid timezone: {task_data.timezone}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Parse run_time
|
||||||
|
hour, minute = map(int, task_data.run_time.split(':'))
|
||||||
|
run_time_obj = time_type(hour=hour, minute=minute)
|
||||||
|
|
||||||
|
# Create task
|
||||||
|
task = ScheduledTask(
|
||||||
|
name=task_data.name,
|
||||||
|
enabled=True,
|
||||||
|
source_type=task_data.source_type,
|
||||||
|
source_name=task_data.source_name,
|
||||||
|
download_mode=task_data.download_mode,
|
||||||
|
limit=task_data.limit,
|
||||||
|
sort=task_data.sort,
|
||||||
|
time_filter="day", # Always "day" for daily scheduled tasks
|
||||||
|
min_score=task_data.min_score,
|
||||||
|
no_dupes=True, # Always true for scheduled tasks
|
||||||
|
simple_check=task_data.simple_check,
|
||||||
|
schedule_frequency="daily",
|
||||||
|
run_time=run_time_obj,
|
||||||
|
timezone=task_data.timezone,
|
||||||
|
auth_state=task_data.auth_state
|
||||||
|
)
|
||||||
|
|
||||||
|
# Calculate next run time
|
||||||
|
task.next_run_at = calculate_next_run(task)
|
||||||
|
|
||||||
|
# Save to database
|
||||||
|
db.add(task)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(task)
|
||||||
|
|
||||||
|
# Schedule the task
|
||||||
|
schedule_task(task)
|
||||||
|
|
||||||
|
logger.info(f"Created scheduled task {task.id}: {task.name}")
|
||||||
|
|
||||||
|
return task.to_dict()
|
||||||
|
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to create scheduled task: {e}", exc_info=True)
|
||||||
|
db.rollback()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
detail=f"Failed to create scheduled task: {str(e)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("", response_model=List[ScheduledTaskResponse])
|
||||||
|
async def list_scheduled_tasks(db: Session = Depends(get_db)):
|
||||||
|
"""
|
||||||
|
Get all scheduled tasks.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
tasks = db.query(ScheduledTask).order_by(ScheduledTask.created_at.desc()).all()
|
||||||
|
return [task.to_dict() for task in tasks]
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to list scheduled tasks: {e}", exc_info=True)
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
detail="Failed to list scheduled tasks"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{task_id}", response_model=ScheduledTaskResponse)
|
||||||
|
async def get_scheduled_task(task_id: str, db: Session = Depends(get_db)):
|
||||||
|
"""
|
||||||
|
Get a specific scheduled task by ID.
|
||||||
|
"""
|
||||||
|
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
|
||||||
|
|
||||||
|
if not task:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
|
detail=f"Task {task_id} not found"
|
||||||
|
)
|
||||||
|
|
||||||
|
return task.to_dict()
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/{task_id}", response_model=ScheduledTaskResponse)
|
||||||
|
async def update_scheduled_task(
|
||||||
|
task_id: str,
|
||||||
|
task_data: ScheduledTaskUpdate,
|
||||||
|
db: Session = Depends(get_db)
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Update a scheduled task.
|
||||||
|
"""
|
||||||
|
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
|
||||||
|
|
||||||
|
if not task:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
|
detail=f"Task {task_id} not found"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Update fields
|
||||||
|
if task_data.name is not None:
|
||||||
|
task.name = task_data.name
|
||||||
|
if task_data.limit is not None:
|
||||||
|
task.limit = task_data.limit
|
||||||
|
if task_data.sort is not None:
|
||||||
|
task.sort = task_data.sort
|
||||||
|
if task_data.min_score is not None:
|
||||||
|
task.min_score = task_data.min_score
|
||||||
|
if task_data.simple_check is not None:
|
||||||
|
task.simple_check = task_data.simple_check
|
||||||
|
|
||||||
|
# Handle run_time update
|
||||||
|
reschedule_needed = False
|
||||||
|
if task_data.run_time is not None:
|
||||||
|
hour, minute = map(int, task_data.run_time.split(':'))
|
||||||
|
task.run_time = time_type(hour=hour, minute=minute)
|
||||||
|
reschedule_needed = True
|
||||||
|
|
||||||
|
# Handle timezone update
|
||||||
|
if task_data.timezone is not None:
|
||||||
|
try:
|
||||||
|
pytz.timezone(task_data.timezone)
|
||||||
|
task.timezone = task_data.timezone
|
||||||
|
reschedule_needed = True
|
||||||
|
except pytz.exceptions.UnknownTimeZoneError:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail=f"Invalid timezone: {task_data.timezone}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Handle enabled status
|
||||||
|
if task_data.enabled is not None and task_data.enabled != task.enabled:
|
||||||
|
task.enabled = task_data.enabled
|
||||||
|
reschedule_needed = True
|
||||||
|
|
||||||
|
# Update next_run_at if needed
|
||||||
|
if reschedule_needed:
|
||||||
|
if task.enabled:
|
||||||
|
task.next_run_at = calculate_next_run(task)
|
||||||
|
# Reschedule
|
||||||
|
unschedule_task(task_id)
|
||||||
|
schedule_task(task)
|
||||||
|
else:
|
||||||
|
# Unschedule if disabled
|
||||||
|
unschedule_task(task_id)
|
||||||
|
task.next_run_at = None
|
||||||
|
|
||||||
|
task.updated_at = datetime.now(pytz.UTC)
|
||||||
|
|
||||||
|
db.commit()
|
||||||
|
db.refresh(task)
|
||||||
|
|
||||||
|
logger.info(f"Updated scheduled task {task_id}")
|
||||||
|
|
||||||
|
return task.to_dict()
|
||||||
|
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to update task {task_id}: {e}", exc_info=True)
|
||||||
|
db.rollback()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
detail=f"Failed to update task: {str(e)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/{task_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||||
|
async def delete_scheduled_task(task_id: str, db: Session = Depends(get_db)):
|
||||||
|
"""
|
||||||
|
Delete a scheduled task.
|
||||||
|
"""
|
||||||
|
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
|
||||||
|
|
||||||
|
if not task:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
|
detail=f"Task {task_id} not found"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Unschedule
|
||||||
|
unschedule_task(task_id)
|
||||||
|
|
||||||
|
# Delete from database (cascade will delete execution history)
|
||||||
|
db.delete(task)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
logger.info(f"Deleted scheduled task {task_id}")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to delete task {task_id}: {e}", exc_info=True)
|
||||||
|
db.rollback()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
detail=f"Failed to delete task: {str(e)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{task_id}/toggle")
|
||||||
|
async def toggle_scheduled_task(task_id: str, db: Session = Depends(get_db)):
|
||||||
|
"""
|
||||||
|
Toggle a task's enabled status.
|
||||||
|
"""
|
||||||
|
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
|
||||||
|
|
||||||
|
if not task:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
|
detail=f"Task {task_id} not found"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Toggle enabled
|
||||||
|
task.enabled = not task.enabled
|
||||||
|
task.updated_at = datetime.now(pytz.UTC)
|
||||||
|
|
||||||
|
if task.enabled:
|
||||||
|
# Re-enable: schedule and calculate next run
|
||||||
|
task.next_run_at = calculate_next_run(task)
|
||||||
|
schedule_task(task)
|
||||||
|
message = "Task enabled"
|
||||||
|
else:
|
||||||
|
# Disable: unschedule
|
||||||
|
unschedule_task(task_id)
|
||||||
|
task.next_run_at = None
|
||||||
|
message = "Task disabled"
|
||||||
|
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
logger.info(f"Toggled task {task_id}: {message}")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"message": message,
|
||||||
|
"enabled": task.enabled
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to toggle task {task_id}: {e}", exc_info=True)
|
||||||
|
db.rollback()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
detail=f"Failed to toggle task: {str(e)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{task_id}/run-now")
|
||||||
|
async def run_task_now(task_id: str, db: Session = Depends(get_db)):
|
||||||
|
"""
|
||||||
|
Manually trigger a scheduled task to run now.
|
||||||
|
Adds it to the queue with high priority.
|
||||||
|
"""
|
||||||
|
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
|
||||||
|
|
||||||
|
if not task:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
|
detail=f"Task {task_id} not found"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Add to queue with priority (goes ahead of scheduled tasks)
|
||||||
|
await task_queue.add_task(task_id, priority=1)
|
||||||
|
|
||||||
|
queue_status = task_queue.get_queue_status()
|
||||||
|
|
||||||
|
logger.info(f"Manually queued task {task_id} for immediate execution")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"message": f"Task queued for execution",
|
||||||
|
"task_id": task_id,
|
||||||
|
"queue_position": queue_status['queue_size'],
|
||||||
|
"currently_running": queue_status['current_task'],
|
||||||
|
"status": "queued" if queue_status['current_task'] else "starting"
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to queue task {task_id}: {e}", exc_info=True)
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
detail=f"Failed to queue task: {str(e)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{task_id}/history", response_model=List[TaskExecutionResponse])
|
||||||
|
async def get_task_history(task_id: str, limit: int = 10, db: Session = Depends(get_db)):
|
||||||
|
"""
|
||||||
|
Get execution history for a specific task.
|
||||||
|
"""
|
||||||
|
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
|
||||||
|
|
||||||
|
if not task:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND,
|
||||||
|
detail=f"Task {task_id} not found"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
executions = db.query(TaskExecutionHistory)\
|
||||||
|
.filter(TaskExecutionHistory.task_id == task_id)\
|
||||||
|
.order_by(TaskExecutionHistory.started_at.desc())\
|
||||||
|
.limit(limit)\
|
||||||
|
.all()
|
||||||
|
|
||||||
|
return [execution.to_dict() for execution in executions]
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to get history for task {task_id}: {e}", exc_info=True)
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
detail="Failed to get task history"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/history/recent", response_model=List[TaskExecutionResponse])
|
||||||
|
async def get_recent_history(limit: int = 20, db: Session = Depends(get_db)):
|
||||||
|
"""
|
||||||
|
Get recent execution history across all tasks.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
executions = db.query(TaskExecutionHistory)\
|
||||||
|
.order_by(TaskExecutionHistory.started_at.desc())\
|
||||||
|
.limit(limit)\
|
||||||
|
.all()
|
||||||
|
|
||||||
|
return [execution.to_dict() for execution in executions]
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to get recent history: {e}", exc_info=True)
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
detail="Failed to get execution history"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/queue/status")
|
||||||
|
async def get_queue_status():
|
||||||
|
"""Get current task queue status"""
|
||||||
|
try:
|
||||||
|
status = task_queue.get_queue_status()
|
||||||
|
|
||||||
|
# Get details of current task if any
|
||||||
|
current_task_info = None
|
||||||
|
if status['current_task']:
|
||||||
|
db = next(get_db())
|
||||||
|
try:
|
||||||
|
task = db.query(ScheduledTask).filter(ScheduledTask.id == status['current_task']).first()
|
||||||
|
if task:
|
||||||
|
current_task_info = {
|
||||||
|
'id': task.id,
|
||||||
|
'name': task.name,
|
||||||
|
'source': f"{task.source_type}/{task.source_name}"
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
return {
|
||||||
|
'queue_size': status['queue_size'],
|
||||||
|
'is_processing': status['is_processing'],
|
||||||
|
'current_task': current_task_info
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to get queue status: {e}", exc_info=True)
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
detail="Failed to get queue status"
|
||||||
|
)
|
||||||
@@ -0,0 +1,360 @@
|
|||||||
|
"""
|
||||||
|
Scheduler service for managing scheduled download tasks.
|
||||||
|
Uses APScheduler with task queue for sequential execution.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||||
|
from apscheduler.triggers.cron import CronTrigger
|
||||||
|
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
|
||||||
|
from datetime import datetime, time as time_type, timedelta
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
import pytz
|
||||||
|
import logging
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
from .database import SessionLocal, DATA_DIR
|
||||||
|
from .models import ScheduledTask, TaskExecutionHistory
|
||||||
|
from .task_queue import task_queue
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Scheduler job store database (separate from main DB)
|
||||||
|
SCHEDULER_DB_PATH = DATA_DIR / "scheduler_jobs.db"
|
||||||
|
|
||||||
|
# Configure job stores
|
||||||
|
jobstores = {
|
||||||
|
'default': SQLAlchemyJobStore(url=f'sqlite:///{SCHEDULER_DB_PATH}')
|
||||||
|
}
|
||||||
|
|
||||||
|
# Configure scheduler for Docker container
|
||||||
|
scheduler = AsyncIOScheduler(
|
||||||
|
jobstores=jobstores,
|
||||||
|
timezone=pytz.UTC, # Container runs in UTC
|
||||||
|
job_defaults={
|
||||||
|
'coalesce': True, # Combine multiple missed executions into one
|
||||||
|
'max_instances': 1, # Only one instance of each job at a time
|
||||||
|
'misfire_grace_time': 3600 # Allow up to 1 hour late execution
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def queue_scheduled_task(task_id: str):
|
||||||
|
"""
|
||||||
|
Called by scheduler at the configured time.
|
||||||
|
Adds task to queue rather than executing immediately.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
task_id: UUID of the scheduled task
|
||||||
|
"""
|
||||||
|
logger.info(f"Scheduler triggered for task {task_id}, adding to queue")
|
||||||
|
try:
|
||||||
|
await task_queue.add_task(task_id, priority=0) # Normal priority for scheduled tasks
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to queue scheduled task {task_id}: {e}", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
def schedule_task(task: ScheduledTask):
|
||||||
|
"""
|
||||||
|
Schedule a task to be added to the queue at specified time.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
task: ScheduledTask model instance
|
||||||
|
"""
|
||||||
|
if not task.enabled:
|
||||||
|
logger.info(f"Skipping scheduling for disabled task {task.id}")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Convert user's timezone to UTC for container execution
|
||||||
|
user_tz = pytz.timezone(task.timezone)
|
||||||
|
|
||||||
|
# Parse time
|
||||||
|
if isinstance(task.run_time, str):
|
||||||
|
hour, minute = map(int, task.run_time.split(':')[:2])
|
||||||
|
else:
|
||||||
|
hour = task.run_time.hour
|
||||||
|
minute = task.run_time.minute
|
||||||
|
|
||||||
|
# Create cron trigger with user's timezone
|
||||||
|
trigger = CronTrigger(
|
||||||
|
hour=hour,
|
||||||
|
minute=minute,
|
||||||
|
timezone=user_tz
|
||||||
|
)
|
||||||
|
|
||||||
|
# Add job to scheduler
|
||||||
|
scheduler.add_job(
|
||||||
|
func=queue_scheduled_task,
|
||||||
|
trigger=trigger,
|
||||||
|
args=[task.id],
|
||||||
|
id=str(task.id),
|
||||||
|
replace_existing=True,
|
||||||
|
name=f"{task.name} ({task.source_type}/{task.source_name})"
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(f"Scheduled task {task.id} '{task.name}' for {hour:02d}:{minute:02d} {task.timezone}")
|
||||||
|
|
||||||
|
# Update next_run_at
|
||||||
|
next_run = trigger.get_next_fire_time(None, datetime.now(user_tz))
|
||||||
|
if next_run:
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
db_task = db.query(ScheduledTask).filter(ScheduledTask.id == task.id).first()
|
||||||
|
if db_task:
|
||||||
|
db_task.next_run_at = next_run
|
||||||
|
db.commit()
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to schedule task {task.id}: {e}", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
def unschedule_task(task_id: str):
|
||||||
|
"""
|
||||||
|
Remove a task from the scheduler.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
task_id: UUID of the scheduled task
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
if scheduler.get_job(task_id):
|
||||||
|
scheduler.remove_job(task_id)
|
||||||
|
logger.info(f"Unscheduled task {task_id}")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to unschedule task {task_id}: {e}", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
def load_scheduled_tasks():
|
||||||
|
"""
|
||||||
|
Load all enabled scheduled tasks from database and schedule them.
|
||||||
|
Called on application startup.
|
||||||
|
"""
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
tasks = db.query(ScheduledTask).filter(ScheduledTask.enabled == True).all()
|
||||||
|
logger.info(f"Loading {len(tasks)} enabled scheduled tasks")
|
||||||
|
|
||||||
|
for task in tasks:
|
||||||
|
schedule_task(task)
|
||||||
|
|
||||||
|
logger.info(f"Loaded {len(tasks)} scheduled tasks")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to load scheduled tasks: {e}", exc_info=True)
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def execute_scheduled_task(task_id: str):
|
||||||
|
"""
|
||||||
|
Execute a scheduled download task.
|
||||||
|
This function BLOCKS until the download is complete,
|
||||||
|
ensuring sequential execution.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
task_id: UUID of the scheduled task
|
||||||
|
"""
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
# Load task from database
|
||||||
|
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
|
||||||
|
|
||||||
|
if not task:
|
||||||
|
logger.error(f"Task {task_id} not found in database")
|
||||||
|
return
|
||||||
|
|
||||||
|
if not task.enabled:
|
||||||
|
logger.info(f"Skipping disabled task {task_id}")
|
||||||
|
# Still record in history that it was skipped
|
||||||
|
execution = TaskExecutionHistory(
|
||||||
|
task_id=task_id,
|
||||||
|
status='skipped',
|
||||||
|
completed_at=datetime.now(pytz.UTC)
|
||||||
|
)
|
||||||
|
db.add(execution)
|
||||||
|
db.commit()
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info(f"Executing scheduled task {task_id}: {task.source_type}/{task.source_name}")
|
||||||
|
|
||||||
|
# Create execution history record
|
||||||
|
execution = TaskExecutionHistory(
|
||||||
|
task_id=task_id,
|
||||||
|
status='running',
|
||||||
|
started_at=datetime.now(pytz.UTC)
|
||||||
|
)
|
||||||
|
db.add(execution)
|
||||||
|
db.commit()
|
||||||
|
execution_id = execution.id
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Import here to avoid circular imports
|
||||||
|
from .main import create_download_with_bdfr_api, bdfr_manager
|
||||||
|
|
||||||
|
# Build download parameters
|
||||||
|
kwargs = {
|
||||||
|
'limit': task.limit,
|
||||||
|
'sort': task.sort,
|
||||||
|
'time_filter': 'day', # Always "day" for daily scheduled tasks
|
||||||
|
'no_dupes': True, # Always enabled for scheduled tasks
|
||||||
|
'simple_check': task.simple_check,
|
||||||
|
'auth_state': task.auth_state
|
||||||
|
}
|
||||||
|
|
||||||
|
# Create download using existing API
|
||||||
|
download_id = await create_download_with_bdfr_api(
|
||||||
|
download_type=task.source_type,
|
||||||
|
name=task.source_name,
|
||||||
|
**kwargs
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(f"Task {task_id} started as download {download_id}")
|
||||||
|
|
||||||
|
# Update execution record with download_id
|
||||||
|
execution = db.query(TaskExecutionHistory).filter(TaskExecutionHistory.id == execution_id).first()
|
||||||
|
if execution:
|
||||||
|
execution.download_id = download_id
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
# **CRITICAL: Wait for download to complete before returning**
|
||||||
|
await wait_for_download_completion(download_id, bdfr_manager)
|
||||||
|
|
||||||
|
# Check final status
|
||||||
|
download_status = bdfr_manager.get_download_status(download_id)
|
||||||
|
|
||||||
|
# Update execution history
|
||||||
|
execution = db.query(TaskExecutionHistory).filter(TaskExecutionHistory.id == execution_id).first()
|
||||||
|
if execution:
|
||||||
|
if download_status and download_status['status'] == 'completed':
|
||||||
|
execution.status = 'success'
|
||||||
|
execution.items_found = download_status.get('items_found', 0)
|
||||||
|
execution.items_downloaded = download_status.get('items_processed', 0)
|
||||||
|
logger.info(f"Task {task_id} completed successfully")
|
||||||
|
else:
|
||||||
|
execution.status = 'failed'
|
||||||
|
execution.error_message = download_status.get('error', 'Unknown error') if download_status else 'Download status not found'
|
||||||
|
logger.error(f"Task {task_id} failed: {execution.error_message}")
|
||||||
|
|
||||||
|
execution.completed_at = datetime.now(pytz.UTC)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
# Update task timestamps
|
||||||
|
task.last_run_at = datetime.now(pytz.UTC)
|
||||||
|
task.next_run_at = calculate_next_run(task)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Task {task_id} execution error: {e}", exc_info=True)
|
||||||
|
|
||||||
|
# Update execution history with error
|
||||||
|
execution = db.query(TaskExecutionHistory).filter(TaskExecutionHistory.id == execution_id).first()
|
||||||
|
if execution:
|
||||||
|
execution.status = 'failed'
|
||||||
|
execution.error_message = str(e)
|
||||||
|
execution.completed_at = datetime.now(pytz.UTC)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
raise
|
||||||
|
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def wait_for_download_completion(download_id: str, bdfr_manager, timeout: int = 3600):
|
||||||
|
"""
|
||||||
|
Wait for a download to complete.
|
||||||
|
Polls the download status until it's no longer running.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
download_id: The download to wait for
|
||||||
|
bdfr_manager: BDFRManager instance
|
||||||
|
timeout: Maximum seconds to wait (default 1 hour)
|
||||||
|
"""
|
||||||
|
start_time = datetime.now()
|
||||||
|
check_interval = 5 # Check every 5 seconds
|
||||||
|
|
||||||
|
logger.info(f"Waiting for download {download_id} to complete...")
|
||||||
|
|
||||||
|
while True:
|
||||||
|
# Check if timeout exceeded
|
||||||
|
elapsed = (datetime.now() - start_time).total_seconds()
|
||||||
|
if elapsed > timeout:
|
||||||
|
logger.error(f"Download {download_id} timed out after {timeout}s")
|
||||||
|
raise TimeoutError(f"Download exceeded timeout of {timeout}s")
|
||||||
|
|
||||||
|
# Check download status
|
||||||
|
status = bdfr_manager.get_download_status(download_id)
|
||||||
|
|
||||||
|
if not status:
|
||||||
|
logger.warning(f"Download {download_id} status not found, assuming complete")
|
||||||
|
break
|
||||||
|
|
||||||
|
download_status = status.get('status', 'unknown')
|
||||||
|
|
||||||
|
# Check if download is finished (completed, failed, or cancelled)
|
||||||
|
if download_status in ['completed', 'failed', 'cancelled']:
|
||||||
|
logger.info(f"Download {download_id} finished with status: {download_status}")
|
||||||
|
break
|
||||||
|
|
||||||
|
# Still running, wait before checking again
|
||||||
|
await asyncio.sleep(check_interval)
|
||||||
|
|
||||||
|
|
||||||
|
def calculate_next_run(task: ScheduledTask) -> datetime:
|
||||||
|
"""
|
||||||
|
Calculate the next run time for a task based on its schedule.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
task: ScheduledTask instance
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Next run datetime in UTC
|
||||||
|
"""
|
||||||
|
user_tz = pytz.timezone(task.timezone)
|
||||||
|
now = datetime.now(user_tz)
|
||||||
|
|
||||||
|
# Parse run time
|
||||||
|
if isinstance(task.run_time, str):
|
||||||
|
hour, minute = map(int, task.run_time.split(':')[:2])
|
||||||
|
else:
|
||||||
|
hour = task.run_time.hour
|
||||||
|
minute = task.run_time.minute
|
||||||
|
|
||||||
|
# Calculate next run
|
||||||
|
next_run = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
|
||||||
|
|
||||||
|
# If time has passed today, schedule for tomorrow
|
||||||
|
if next_run <= now:
|
||||||
|
next_run += timedelta(days=1)
|
||||||
|
|
||||||
|
# Convert to UTC
|
||||||
|
return next_run.astimezone(pytz.UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def start_scheduler():
|
||||||
|
"""Start the scheduler. Called on application startup."""
|
||||||
|
try:
|
||||||
|
if not scheduler.running:
|
||||||
|
scheduler.start()
|
||||||
|
logger.info("Scheduler started")
|
||||||
|
|
||||||
|
# Set execute callback for task queue
|
||||||
|
task_queue.set_execute_callback(execute_scheduled_task)
|
||||||
|
|
||||||
|
# Load existing tasks
|
||||||
|
load_scheduled_tasks()
|
||||||
|
else:
|
||||||
|
logger.info("Scheduler already running")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to start scheduler: {e}", exc_info=True)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def stop_scheduler():
|
||||||
|
"""Stop the scheduler. Called on application shutdown."""
|
||||||
|
try:
|
||||||
|
if scheduler.running:
|
||||||
|
scheduler.shutdown(wait=True)
|
||||||
|
logger.info("Scheduler stopped")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to stop scheduler: {e}", exc_info=True)
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
"""
|
||||||
|
Task Queue Manager for sequential execution of scheduled downloads.
|
||||||
|
Ensures only one task runs at a time.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from typing import Optional, Dict
|
||||||
|
from datetime import datetime
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class TaskQueue:
|
||||||
|
"""
|
||||||
|
Manages sequential execution of scheduled download tasks.
|
||||||
|
Ensures only one task runs at a time.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.queue: asyncio.Queue = asyncio.Queue()
|
||||||
|
self.current_task: Optional[str] = None # Current task_id being executed
|
||||||
|
self.is_processing: bool = False
|
||||||
|
self.worker_task: Optional[asyncio.Task] = None
|
||||||
|
self._execute_callback = None
|
||||||
|
|
||||||
|
def set_execute_callback(self, callback):
|
||||||
|
"""
|
||||||
|
Set the callback function to execute tasks.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
callback: Async function that takes task_id and executes it
|
||||||
|
"""
|
||||||
|
self._execute_callback = callback
|
||||||
|
|
||||||
|
async def add_task(self, task_id: str, priority: int = 0):
|
||||||
|
"""
|
||||||
|
Add a task to the queue.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
task_id: UUID of the scheduled task
|
||||||
|
priority: 0 = scheduled (normal), 1 = manual "Run Now" (higher priority)
|
||||||
|
"""
|
||||||
|
task_info = {
|
||||||
|
'task_id': task_id,
|
||||||
|
'priority': priority,
|
||||||
|
'queued_at': datetime.now()
|
||||||
|
}
|
||||||
|
|
||||||
|
# For priority tasks, we need to reorder the queue
|
||||||
|
if priority > 0:
|
||||||
|
# Get all items from queue
|
||||||
|
items = []
|
||||||
|
while not self.queue.empty():
|
||||||
|
try:
|
||||||
|
items.append(await asyncio.wait_for(self.queue.get(), timeout=0.1))
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
break
|
||||||
|
|
||||||
|
# Add new priority task
|
||||||
|
await self.queue.put(task_info)
|
||||||
|
|
||||||
|
# Re-add other items
|
||||||
|
for item in items:
|
||||||
|
await self.queue.put(item)
|
||||||
|
|
||||||
|
logger.info(f"Priority task {task_id} added to front of queue (queue_size={self.queue.qsize()})")
|
||||||
|
else:
|
||||||
|
await self.queue.put(task_info)
|
||||||
|
logger.info(f"Task {task_id} added to queue (priority={priority}, queue_size={self.queue.qsize()})")
|
||||||
|
|
||||||
|
# Start worker if not already running
|
||||||
|
if not self.is_processing:
|
||||||
|
await self.start_worker()
|
||||||
|
|
||||||
|
async def start_worker(self):
|
||||||
|
"""Start the queue worker if not already running"""
|
||||||
|
if self.worker_task is None or self.worker_task.done():
|
||||||
|
self.worker_task = asyncio.create_task(self._process_queue())
|
||||||
|
logger.info("Queue worker started")
|
||||||
|
|
||||||
|
async def _process_queue(self):
|
||||||
|
"""Process tasks from queue sequentially"""
|
||||||
|
self.is_processing = True
|
||||||
|
logger.info("Queue worker processing started")
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
# Wait for next task (with timeout to allow graceful shutdown)
|
||||||
|
try:
|
||||||
|
task_info = await asyncio.wait_for(
|
||||||
|
self.queue.get(),
|
||||||
|
timeout=60.0
|
||||||
|
)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
# Check if queue is empty
|
||||||
|
if self.queue.empty():
|
||||||
|
logger.info("Queue empty, worker stopping")
|
||||||
|
break
|
||||||
|
continue
|
||||||
|
|
||||||
|
task_id = task_info['task_id']
|
||||||
|
self.current_task = task_id
|
||||||
|
|
||||||
|
logger.info(f"Executing task {task_id} from queue (queue_size={self.queue.qsize()})")
|
||||||
|
|
||||||
|
# Execute the task (this will block until download completes)
|
||||||
|
try:
|
||||||
|
if self._execute_callback:
|
||||||
|
await self._execute_callback(task_id)
|
||||||
|
logger.info(f"Task {task_id} completed successfully")
|
||||||
|
else:
|
||||||
|
logger.error(f"No execute callback set, cannot run task {task_id}")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Task {task_id} failed: {e}", exc_info=True)
|
||||||
|
finally:
|
||||||
|
self.current_task = None
|
||||||
|
self.queue.task_done()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Queue worker error: {e}", exc_info=True)
|
||||||
|
|
||||||
|
self.is_processing = False
|
||||||
|
logger.info("Queue worker stopped")
|
||||||
|
|
||||||
|
def get_queue_status(self) -> Dict:
|
||||||
|
"""Get current queue status"""
|
||||||
|
return {
|
||||||
|
'current_task': self.current_task,
|
||||||
|
'queue_size': self.queue.qsize(),
|
||||||
|
'is_processing': self.is_processing
|
||||||
|
}
|
||||||
|
|
||||||
|
async def stop(self):
|
||||||
|
"""Stop the queue worker gracefully"""
|
||||||
|
logger.info("Stopping queue worker...")
|
||||||
|
if self.worker_task and not self.worker_task.done():
|
||||||
|
# Wait for current task to complete
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(self.worker_task, timeout=300) # 5 minute timeout
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
logger.warning("Queue worker did not stop within timeout, cancelling")
|
||||||
|
self.worker_task.cancel()
|
||||||
|
logger.info("Queue worker stopped")
|
||||||
|
|
||||||
|
|
||||||
|
# Global queue instance
|
||||||
|
task_queue = TaskQueue()
|
||||||
Binary file not shown.
Binary file not shown.
@@ -8,4 +8,8 @@ python-dotenv>=1.0.0
|
|||||||
pydantic>=2.0.0
|
pydantic>=2.0.0
|
||||||
pydantic-settings>=2.0.0
|
pydantic-settings>=2.0.0
|
||||||
requests>=2.25.0
|
requests>=2.25.0
|
||||||
httpx>=0.24.0
|
httpx>=0.24.0
|
||||||
|
sqlalchemy>=2.0.0
|
||||||
|
alembic>=1.12.0
|
||||||
|
apscheduler>=3.10.0
|
||||||
|
pytz>=2023.3
|
||||||
@@ -203,6 +203,14 @@ main {
|
|||||||
border-top: 1px solid #e9ecef;
|
border-top: 1px solid #e9ecef;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.schedule-options {
|
||||||
|
margin-top: 20px;
|
||||||
|
padding: 20px;
|
||||||
|
background: white;
|
||||||
|
border-radius: 8px;
|
||||||
|
border: 2px solid #667eea;
|
||||||
|
}
|
||||||
|
|
||||||
.form-section h4 {
|
.form-section h4 {
|
||||||
color: #2c3e50;
|
color: #2c3e50;
|
||||||
font-size: 1.1rem;
|
font-size: 1.1rem;
|
||||||
@@ -770,6 +778,239 @@ footer {
|
|||||||
opacity: 0.8;
|
opacity: 0.8;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Scheduled Downloads Section */
|
||||||
|
.scheduled-section {
|
||||||
|
margin-bottom: 40px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.scheduled-header {
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
align-items: center;
|
||||||
|
margin-bottom: 20px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.scheduled-header h2 {
|
||||||
|
color: #2c3e50;
|
||||||
|
font-size: 1.8rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.queue-status {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 10px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.queue-badge {
|
||||||
|
background: rgba(102, 126, 234, 0.1);
|
||||||
|
color: #667eea;
|
||||||
|
padding: 8px 16px;
|
||||||
|
border-radius: 20px;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
font-weight: 600;
|
||||||
|
border: 2px solid rgba(102, 126, 234, 0.3);
|
||||||
|
}
|
||||||
|
|
||||||
|
.scheduled-container {
|
||||||
|
background: #f8f9fa;
|
||||||
|
border-radius: 10px;
|
||||||
|
padding: 20px;
|
||||||
|
min-height: 100px;
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.no-tasks {
|
||||||
|
text-align: center;
|
||||||
|
color: #666;
|
||||||
|
}
|
||||||
|
|
||||||
|
.scheduled-list {
|
||||||
|
background: #f8f9fa;
|
||||||
|
border-radius: 10px;
|
||||||
|
padding: 20px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.scheduled-items {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 15px;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Scheduled Task Card */
|
||||||
|
.task-card {
|
||||||
|
background: white;
|
||||||
|
padding: 20px;
|
||||||
|
border-radius: 8px;
|
||||||
|
border-left: 4px solid #667eea;
|
||||||
|
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||||||
|
transition: all 0.3s ease;
|
||||||
|
}
|
||||||
|
|
||||||
|
.task-card:hover {
|
||||||
|
box-shadow: 0 4px 12px rgba(0,0,0,0.15);
|
||||||
|
transform: translateY(-2px);
|
||||||
|
}
|
||||||
|
|
||||||
|
.task-card.disabled {
|
||||||
|
opacity: 0.6;
|
||||||
|
border-left-color: #6c757d;
|
||||||
|
}
|
||||||
|
|
||||||
|
.task-header {
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
align-items: flex-start;
|
||||||
|
margin-bottom: 15px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.task-info h4 {
|
||||||
|
color: #2c3e50;
|
||||||
|
margin-bottom: 8px;
|
||||||
|
font-size: 1.1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.task-meta {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 4px;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
color: #666;
|
||||||
|
}
|
||||||
|
|
||||||
|
.task-meta-item {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 6px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.task-meta-item strong {
|
||||||
|
color: #555;
|
||||||
|
min-width: 80px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.task-status {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-badge {
|
||||||
|
font-weight: 600;
|
||||||
|
padding: 6px 12px;
|
||||||
|
border-radius: 20px;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-badge.enabled {
|
||||||
|
background: #d4edda;
|
||||||
|
color: #155724;
|
||||||
|
border: 1px solid #c3e6cb;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-badge.disabled {
|
||||||
|
background: #f8d7da;
|
||||||
|
color: #721c24;
|
||||||
|
border: 1px solid #f5c6cb;
|
||||||
|
}
|
||||||
|
|
||||||
|
.task-controls {
|
||||||
|
display: flex;
|
||||||
|
gap: 8px;
|
||||||
|
margin-top: 15px;
|
||||||
|
padding-top: 15px;
|
||||||
|
border-top: 1px solid #e9ecef;
|
||||||
|
}
|
||||||
|
|
||||||
|
.btn-task {
|
||||||
|
padding: 8px 16px;
|
||||||
|
border: none;
|
||||||
|
border-radius: 6px;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
font-weight: 600;
|
||||||
|
cursor: pointer;
|
||||||
|
transition: all 0.2s ease;
|
||||||
|
text-transform: uppercase;
|
||||||
|
letter-spacing: 0.5px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.btn-toggle {
|
||||||
|
background: #667eea;
|
||||||
|
color: white;
|
||||||
|
}
|
||||||
|
|
||||||
|
.btn-toggle:hover {
|
||||||
|
background: #5568d3;
|
||||||
|
transform: translateY(-1px);
|
||||||
|
box-shadow: 0 2px 8px rgba(102, 126, 234, 0.3);
|
||||||
|
}
|
||||||
|
|
||||||
|
.btn-toggle.disabled {
|
||||||
|
background: #28a745;
|
||||||
|
}
|
||||||
|
|
||||||
|
.btn-toggle.disabled:hover {
|
||||||
|
background: #218838;
|
||||||
|
}
|
||||||
|
|
||||||
|
.btn-run {
|
||||||
|
background: #ffc107;
|
||||||
|
color: #000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.btn-run:hover {
|
||||||
|
background: #ffca2c;
|
||||||
|
transform: translateY(-1px);
|
||||||
|
box-shadow: 0 2px 8px rgba(255, 193, 7, 0.3);
|
||||||
|
}
|
||||||
|
|
||||||
|
.btn-delete {
|
||||||
|
background: #dc3545;
|
||||||
|
color: white;
|
||||||
|
}
|
||||||
|
|
||||||
|
.btn-delete:hover {
|
||||||
|
background: #c82333;
|
||||||
|
transform: translateY(-1px);
|
||||||
|
box-shadow: 0 2px 8px rgba(220, 53, 69, 0.3);
|
||||||
|
}
|
||||||
|
|
||||||
|
.btn-history {
|
||||||
|
background: #17a2b8;
|
||||||
|
color: white;
|
||||||
|
}
|
||||||
|
|
||||||
|
.btn-history:hover {
|
||||||
|
background: #138496;
|
||||||
|
transform: translateY(-1px);
|
||||||
|
box-shadow: 0 2px 8px rgba(23, 162, 184, 0.3);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Task Schedule Info */
|
||||||
|
.task-schedule {
|
||||||
|
background: rgba(102, 126, 234, 0.05);
|
||||||
|
padding: 12px;
|
||||||
|
border-radius: 6px;
|
||||||
|
margin-bottom: 10px;
|
||||||
|
border-left: 3px solid #667eea;
|
||||||
|
}
|
||||||
|
|
||||||
|
.task-schedule-item {
|
||||||
|
font-size: 0.9rem;
|
||||||
|
color: #555;
|
||||||
|
margin-bottom: 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.task-schedule-item:last-child {
|
||||||
|
margin-bottom: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.task-schedule-item strong {
|
||||||
|
color: #2c3e50;
|
||||||
|
}
|
||||||
|
|
||||||
/* Animations */
|
/* Animations */
|
||||||
@keyframes pulse {
|
@keyframes pulse {
|
||||||
0% { box-shadow: 0 2px 8px rgba(0,0,0,0.1); }
|
0% { box-shadow: 0 2px 8px rgba(0,0,0,0.1); }
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ class BDFRApp {
|
|||||||
this.maxReconnectAttempts = 5;
|
this.maxReconnectAttempts = 5;
|
||||||
this.reconnectDelay = 1000;
|
this.reconnectDelay = 1000;
|
||||||
this.downloads = new Map();
|
this.downloads = new Map();
|
||||||
|
this.scheduledTasks = new Map();
|
||||||
this.authState = null;
|
this.authState = null;
|
||||||
this.authenticated = false;
|
this.authenticated = false;
|
||||||
|
|
||||||
@@ -19,6 +20,10 @@ class BDFRApp {
|
|||||||
// Check for stored auth state first
|
// Check for stored auth state first
|
||||||
this.authState = this.getStoredAuthState();
|
this.authState = this.getStoredAuthState();
|
||||||
this.checkAuthentication();
|
this.checkAuthentication();
|
||||||
|
|
||||||
|
// Load scheduled tasks
|
||||||
|
this.loadScheduledTasks();
|
||||||
|
this.startQueuePolling();
|
||||||
}
|
}
|
||||||
|
|
||||||
initializeElements() {
|
initializeElements() {
|
||||||
@@ -32,6 +37,13 @@ class BDFRApp {
|
|||||||
this.downloadsList = document.getElementById('downloadsList');
|
this.downloadsList = document.getElementById('downloadsList');
|
||||||
this.downloadsItems = document.getElementById('downloadsItems');
|
this.downloadsItems = document.getElementById('downloadsItems');
|
||||||
|
|
||||||
|
// Scheduled tasks containers
|
||||||
|
this.scheduledContainer = document.getElementById('scheduledContainer');
|
||||||
|
this.scheduledList = document.getElementById('scheduledList');
|
||||||
|
this.scheduledItems = document.getElementById('scheduledItems');
|
||||||
|
this.queueStatus = document.getElementById('queueStatus');
|
||||||
|
this.queueCount = document.getElementById('queueCount');
|
||||||
|
|
||||||
// Status elements
|
// Status elements
|
||||||
this.wsStatus = document.getElementById('wsStatus');
|
this.wsStatus = document.getElementById('wsStatus');
|
||||||
this.bdfrStatus = document.getElementById('bdfrStatus');
|
this.bdfrStatus = document.getElementById('bdfrStatus');
|
||||||
@@ -44,6 +56,12 @@ class BDFRApp {
|
|||||||
this.logoutBtn = document.getElementById('logoutBtn');
|
this.logoutBtn = document.getElementById('logoutBtn');
|
||||||
this.authStateInput = document.getElementById('authState');
|
this.authStateInput = document.getElementById('authState');
|
||||||
this.userAuthStateInput = document.getElementById('userAuthState');
|
this.userAuthStateInput = document.getElementById('userAuthState');
|
||||||
|
|
||||||
|
// Scheduled task form elements
|
||||||
|
this.runDailyCheckbox = document.getElementById('runDaily');
|
||||||
|
this.scheduleOptions = document.getElementById('scheduleOptions');
|
||||||
|
this.taskNameInput = document.getElementById('taskName');
|
||||||
|
this.runTimeInput = document.getElementById('runTime');
|
||||||
}
|
}
|
||||||
|
|
||||||
bindEvents() {
|
bindEvents() {
|
||||||
@@ -57,6 +75,21 @@ class BDFRApp {
|
|||||||
radio.addEventListener('change', (e) => this.updateSourceTypeUI(e.target.value));
|
radio.addEventListener('change', (e) => this.updateSourceTypeUI(e.target.value));
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Run Daily checkbox toggle
|
||||||
|
if (this.runDailyCheckbox) {
|
||||||
|
this.runDailyCheckbox.addEventListener('change', (e) => {
|
||||||
|
this.scheduleOptions.style.display = e.target.checked ? 'block' : 'none';
|
||||||
|
if (e.target.checked && !this.taskNameInput.value) {
|
||||||
|
// Auto-generate task name
|
||||||
|
const sourceType = document.querySelector('input[name="source_type"]:checked').value;
|
||||||
|
const sourceName = document.getElementById('sourceName').value.trim();
|
||||||
|
if (sourceName) {
|
||||||
|
this.taskNameInput.value = `Daily ${sourceName} ${sourceType}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
if (this.subredditForm) {
|
if (this.subredditForm) {
|
||||||
this.subredditForm.addEventListener('submit', (e) => this.handleSubredditSubmit(e));
|
this.subredditForm.addEventListener('submit', (e) => this.handleSubredditSubmit(e));
|
||||||
@@ -292,6 +325,7 @@ class BDFRApp {
|
|||||||
const sort = formData.get('sort');
|
const sort = formData.get('sort');
|
||||||
const noDupes = document.getElementById('noDupes').checked;
|
const noDupes = document.getElementById('noDupes').checked;
|
||||||
const simpleCheck = document.getElementById('simpleCheck').checked;
|
const simpleCheck = document.getElementById('simpleCheck').checked;
|
||||||
|
const runDaily = this.runDailyCheckbox.checked;
|
||||||
|
|
||||||
// Validate source name
|
// Validate source name
|
||||||
if (sourceType === 'subreddit' && !this.validateSubreddit(sourceName)) {
|
if (sourceType === 'subreddit' && !this.validateSubreddit(sourceName)) {
|
||||||
@@ -303,6 +337,12 @@ class BDFRApp {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If Run Daily is checked, create scheduled task instead
|
||||||
|
if (runDaily) {
|
||||||
|
await this.createScheduledTask(e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Build confirmation message
|
// Build confirmation message
|
||||||
const modeLabels = {
|
const modeLabels = {
|
||||||
'download': 'Download (media files)',
|
'download': 'Download (media files)',
|
||||||
@@ -994,6 +1034,275 @@ class BDFRApp {
|
|||||||
this.showError('Authentication failed');
|
this.showError('Authentication failed');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ===== Scheduled Tasks Methods =====
|
||||||
|
|
||||||
|
async loadScheduledTasks() {
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/scheduled-tasks');
|
||||||
|
const tasks = await response.json();
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
this.scheduledTasks.clear();
|
||||||
|
tasks.forEach(task => {
|
||||||
|
this.scheduledTasks.set(task.id, task);
|
||||||
|
});
|
||||||
|
this.renderScheduledTasks();
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to load scheduled tasks:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
renderScheduledTasks() {
|
||||||
|
if (this.scheduledTasks.size > 0) {
|
||||||
|
this.scheduledContainer.style.display = 'none';
|
||||||
|
this.scheduledList.style.display = 'block';
|
||||||
|
this.scheduledItems.innerHTML = '';
|
||||||
|
|
||||||
|
this.scheduledTasks.forEach(task => {
|
||||||
|
const card = this.createTaskCard(task);
|
||||||
|
this.scheduledItems.appendChild(card);
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
this.scheduledContainer.style.display = 'flex';
|
||||||
|
this.scheduledList.style.display = 'none';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
createTaskCard(task) {
|
||||||
|
const card = document.createElement('div');
|
||||||
|
card.className = `task-card ${task.enabled ? '' : 'disabled'}`;
|
||||||
|
card.id = `task-${task.id}`;
|
||||||
|
|
||||||
|
const sourceLabel = task.source_type === 'subreddit' ? `r/${task.source_name}` : `u/${task.source_name}`;
|
||||||
|
const modeLabel = task.download_mode.charAt(0).toUpperCase() + task.download_mode.slice(1);
|
||||||
|
|
||||||
|
const lastRun = task.last_run_at ? new Date(task.last_run_at).toLocaleString() : 'Never';
|
||||||
|
const nextRun = task.next_run_at ? new Date(task.next_run_at).toLocaleString() : 'Not scheduled';
|
||||||
|
|
||||||
|
card.innerHTML = `
|
||||||
|
<div class="task-header">
|
||||||
|
<div class="task-info">
|
||||||
|
<h4>${task.name}</h4>
|
||||||
|
<div class="task-meta">
|
||||||
|
<div class="task-meta-item">
|
||||||
|
<strong>Source:</strong> ${sourceLabel}
|
||||||
|
</div>
|
||||||
|
<div class="task-meta-item">
|
||||||
|
<strong>Mode:</strong> ${modeLabel}
|
||||||
|
</div>
|
||||||
|
<div class="task-meta-item">
|
||||||
|
<strong>Schedule:</strong> Daily at ${task.run_time}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="task-status">
|
||||||
|
<span class="status-badge ${task.enabled ? 'enabled' : 'disabled'}">
|
||||||
|
${task.enabled ? '✓ Enabled' : '✗ Disabled'}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="task-schedule">
|
||||||
|
<div class="task-schedule-item"><strong>Last Run:</strong> ${lastRun}</div>
|
||||||
|
<div class="task-schedule-item"><strong>Next Run:</strong> ${nextRun}</div>
|
||||||
|
</div>
|
||||||
|
<div class="task-controls">
|
||||||
|
<button class="btn-task btn-toggle ${task.enabled ? '' : 'disabled'}" data-task-id="${task.id}" data-action="toggle">
|
||||||
|
${task.enabled ? 'Disable' : 'Enable'}
|
||||||
|
</button>
|
||||||
|
<button class="btn-task btn-run" data-task-id="${task.id}" data-action="run">
|
||||||
|
Run Now
|
||||||
|
</button>
|
||||||
|
<button class="btn-task btn-delete" data-task-id="${task.id}" data-action="delete">
|
||||||
|
Delete
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
|
||||||
|
// Add event listeners to buttons
|
||||||
|
const toggleBtn = card.querySelector('[data-action="toggle"]');
|
||||||
|
const runBtn = card.querySelector('[data-action="run"]');
|
||||||
|
const deleteBtn = card.querySelector('[data-action="delete"]');
|
||||||
|
|
||||||
|
if (toggleBtn) {
|
||||||
|
toggleBtn.addEventListener('click', () => this.toggleTask(task.id));
|
||||||
|
}
|
||||||
|
if (runBtn) {
|
||||||
|
runBtn.addEventListener('click', () => this.runTaskNow(task.id));
|
||||||
|
}
|
||||||
|
if (deleteBtn) {
|
||||||
|
deleteBtn.addEventListener('click', () => this.deleteTask(task.id));
|
||||||
|
}
|
||||||
|
|
||||||
|
return card;
|
||||||
|
}
|
||||||
|
|
||||||
|
async createScheduledTask(e) {
|
||||||
|
const formData = new FormData(e.target);
|
||||||
|
const taskName = this.taskNameInput.value.trim();
|
||||||
|
const runTime = this.runTimeInput.value;
|
||||||
|
|
||||||
|
if (!taskName) {
|
||||||
|
this.showError('Please enter a task name');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!runTime) {
|
||||||
|
this.showError('Please select a run time');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const downloadMode = formData.get('download_mode');
|
||||||
|
const sourceType = formData.get('source_type');
|
||||||
|
const sourceName = formData.get('source_name').trim();
|
||||||
|
const limit = formData.get('limit');
|
||||||
|
const sort = formData.get('sort');
|
||||||
|
|
||||||
|
// Get browser timezone
|
||||||
|
const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
||||||
|
|
||||||
|
const taskData = {
|
||||||
|
name: taskName,
|
||||||
|
source_type: sourceType,
|
||||||
|
source_name: sourceName,
|
||||||
|
download_mode: downloadMode,
|
||||||
|
limit: parseInt(limit),
|
||||||
|
sort: sort,
|
||||||
|
run_time: runTime,
|
||||||
|
timezone: timezone,
|
||||||
|
enabled: true
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
this.showLoading(e.target.querySelector('button'));
|
||||||
|
|
||||||
|
const response = await fetch('/api/scheduled-tasks', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json'
|
||||||
|
},
|
||||||
|
body: JSON.stringify(taskData)
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await response.json();
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
this.showSuccess(`Scheduled task created: ${taskName}`);
|
||||||
|
e.target.reset();
|
||||||
|
this.runDailyCheckbox.checked = false;
|
||||||
|
this.scheduleOptions.style.display = 'none';
|
||||||
|
await this.loadScheduledTasks();
|
||||||
|
} else {
|
||||||
|
this.showError(result.detail || 'Failed to create scheduled task');
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error:', error);
|
||||||
|
this.showError('Network error occurred');
|
||||||
|
} finally {
|
||||||
|
this.hideLoading(e.target.querySelector('button'));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async toggleTask(taskId) {
|
||||||
|
try {
|
||||||
|
const response = await fetch(`/api/scheduled-tasks/${taskId}/toggle`, {
|
||||||
|
method: 'POST'
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await response.json();
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
this.showSuccess(`Task ${result.enabled ? 'enabled' : 'disabled'}`);
|
||||||
|
await this.loadScheduledTasks();
|
||||||
|
} else {
|
||||||
|
this.showError(result.detail || 'Failed to toggle task');
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error:', error);
|
||||||
|
this.showError('Failed to toggle task');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async runTaskNow(taskId) {
|
||||||
|
if (!confirm('Run this scheduled task now?')) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(`/api/scheduled-tasks/${taskId}/run-now`, {
|
||||||
|
method: 'POST'
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await response.json();
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
this.showSuccess('Task added to queue');
|
||||||
|
await this.loadScheduledTasks();
|
||||||
|
} else {
|
||||||
|
this.showError(result.detail || 'Failed to queue task');
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error:', error);
|
||||||
|
this.showError('Failed to queue task');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async deleteTask(taskId) {
|
||||||
|
if (!confirm('Delete this scheduled task? This cannot be undone.')) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(`/api/scheduled-tasks/${taskId}`, {
|
||||||
|
method: 'DELETE'
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
this.showSuccess('Task deleted');
|
||||||
|
await this.loadScheduledTasks();
|
||||||
|
} else {
|
||||||
|
const result = await response.json();
|
||||||
|
this.showError(result.detail || 'Failed to delete task');
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error:', error);
|
||||||
|
this.showError('Failed to delete task');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async updateQueueStatus() {
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/scheduled-tasks/queue/status');
|
||||||
|
const status = await response.json();
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
const queueSize = status.queue_size || 0;
|
||||||
|
if (queueSize > 0 || status.current_task) {
|
||||||
|
this.queueStatus.style.display = 'block';
|
||||||
|
this.queueCount.textContent = queueSize;
|
||||||
|
} else {
|
||||||
|
this.queueStatus.style.display = 'none';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to update queue status:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
startQueuePolling() {
|
||||||
|
// Update queue status every 10 seconds
|
||||||
|
setInterval(() => {
|
||||||
|
this.updateQueueStatus();
|
||||||
|
}, 10000);
|
||||||
|
|
||||||
|
// Initial update
|
||||||
|
this.updateQueueStatus();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize the application when DOM is loaded
|
// Initialize the application when DOM is loaded
|
||||||
|
|||||||
@@ -151,6 +151,26 @@
|
|||||||
<span class="checkmark"></span>
|
<span class="checkmark"></span>
|
||||||
Create Hard Links
|
Create Hard Links
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
|
<label class="checkbox-label">
|
||||||
|
<input type="checkbox" id="runDaily" name="run_daily">
|
||||||
|
<span class="checkmark"></span>
|
||||||
|
Run Daily (Scheduled Download)
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Scheduled Download Options (shown when Run Daily is checked) -->
|
||||||
|
<div id="scheduleOptions" class="schedule-options" style="display: none;">
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="taskName">Task Name:</label>
|
||||||
|
<input type="text" id="taskName" name="task_name" placeholder="e.g., Daily Python Posts">
|
||||||
|
<small class="form-help">A friendly name to identify this scheduled task</small>
|
||||||
|
</div>
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="runTime">Run Time (24-hour format):</label>
|
||||||
|
<input type="time" id="runTime" name="run_time" value="02:00">
|
||||||
|
<small class="form-help">Time to run the download daily (in your local timezone)</small>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -183,6 +203,30 @@
|
|||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
<!-- Scheduled Downloads Section -->
|
||||||
|
<section class="scheduled-section">
|
||||||
|
<div class="scheduled-header">
|
||||||
|
<h2>📅 Scheduled Downloads</h2>
|
||||||
|
<div class="queue-status" id="queueStatus" style="display: none;">
|
||||||
|
<span class="queue-badge">⏳ Queue: <span id="queueCount">0</span></span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div id="scheduledContainer" class="scheduled-container">
|
||||||
|
<div class="no-tasks">
|
||||||
|
<div class="empty-state">
|
||||||
|
<div class="empty-icon">📅</div>
|
||||||
|
<p>No scheduled downloads</p>
|
||||||
|
<p>Check "Run Daily" above to schedule automatic downloads.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Scheduled Tasks List -->
|
||||||
|
<div id="scheduledList" class="scheduled-list" style="display: none;">
|
||||||
|
<div id="scheduledItems" class="scheduled-items"></div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
<!-- Status Section -->
|
<!-- Status Section -->
|
||||||
<section class="status-section">
|
<section class="status-section">
|
||||||
<div class="status-card">
|
<div class="status-card">
|
||||||
|
|||||||
Reference in New Issue
Block a user