feat(UI): added docker image and config

This commit is contained in:
2025-10-10 11:15:28 +13:00
parent 9e61d18bf6
commit 9f5a25fcf5
23 changed files with 3287 additions and 18 deletions
+112
View File
@@ -0,0 +1,112 @@
# Python
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
pip-log.txt
pip-delete-this-directory.txt
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Virtual environments
env/
venv/
ENV/
env.bak/
venv.bak/
.venv/
# IDE
.vscode/
.idea/
*.swp
*.swo
*~
.DS_Store
# Git
.git/
.gitignore
.gitattributes
.gitmodules
# Documentation and configs not needed in image
*.md
!README.md
docs/
.pre-commit-config.yaml
.markdown_style.rb
.mdlrc
_config.yml
SCHEDULED_DOWNLOADS_PLAN.md
# Test files
tests/
test_*.py
*_test.py
conftest.py
# CI/CD
.github/
tox.ini
# Scripts and dev tools
scripts/
devscripts/
# Local data and logs (these should be mounted as volumes)
web_interface/data/*.db
web_interface/data/*.db-*
*.log
*.sqlite
*.sqlite3
# Environment files (use docker-compose env instead)
.env
.env.local
.env.*.local
# Docker files (don't need docker in docker)
Dockerfile*
docker-compose*.yml
.dockerignore
# OS specific
Thumbs.db
ehthumbs.db
Desktop.ini
$RECYCLE.BIN/
# Temporary files
*.tmp
*.temp
*.bak
*.backup
+129
View File
@@ -0,0 +1,129 @@
# BDFR Web Interface - Docker Environment Configuration
# Copy this file to .env and update the values as needed
# ============================================================================
# REDDIT OAUTH CONFIGURATION (REQUIRED for authenticated features)
# ============================================================================
# Get these credentials from: https://www.reddit.com/prefs/apps
#
# Steps to get OAuth credentials:
# 1. Go to https://www.reddit.com/prefs/apps
# 2. Click "Create App" or "Create Another App"
# 3. Fill in:
# - Name: BDFR Web Interface (or your choice)
# - App type: Select "web app"
# - Redirect URI: http://localhost:8000/auth/callback
# 4. Copy the client ID (under the app name) and client secret
BDFR_CLIENT_ID=your_client_id_here
BDFR_CLIENT_SECRET=your_client_secret_here
# IMPORTANT: This MUST match your Reddit OAuth app's redirect URI exactly
# If you change the port, update this accordingly
BDFR_REDIRECT_URI=http://localhost:8000/auth/callback
# ============================================================================
# SERVER CONFIGURATION (Optional)
# ============================================================================
# Host to bind to (0.0.0.0 for all interfaces, 127.0.0.1 for localhost only)
HOST=0.0.0.0
# Port for the web interface
PORT=8000
# Enable debug mode (true/false) - DO NOT use in production
DEBUG=false
# ============================================================================
# DOWNLOAD CONFIGURATION (Optional - Docker handles these by default)
# ============================================================================
# Directory where downloads will be stored (inside container)
# This is mounted from ./downloads on the host
BDFR_DOWNLOAD_DIR=/downloads
# Directory for application data and databases (inside container)
# This is mounted from ./data on the host
BDFR_DATA_DIR=/app/data
# ============================================================================
# BDFR ADVANCED CONFIGURATION (Optional)
# ============================================================================
# BDFR configuration directory (inside container)
# Default: /app/data/bdfr-config (stored in mounted data volume)
# BDFR_CONFIG_DIR=/app/data/bdfr-config
# Maximum wait time for rate limiting (in seconds)
# Default: 120
# BDFR_MAX_WAIT_TIME=120
# Time format for {DATE} in filenames
# Default: ISO 8601 format (%Y-%m-%dT%H:%M:%S)
# BDFR_TIME_FORMAT=%Y-%m-%dT%H:%M:%S
# File naming scheme
# Available variables: {REDDITOR}, {SUBREDDIT}, {POSTID}, {UPVOTES}, {TITLE}, {DATE}, {FLAIR}
# Default: {REDDITOR}_{TITLE}_{POSTID}
# BDFR_FILE_SCHEME={REDDITOR}_{TITLE}_{POSTID}
# Folder naming scheme
# Default: {SUBREDDIT}
# BDFR_FOLDER_SCHEME={SUBREDDIT}
# Environment variables for BDFR config location
# These tell BDFR where to store its configuration files
# APPDATA=/app/data/bdfr-config
# XDG_CONFIG_HOME=/app/data/bdfr-config
# ============================================================================
# DATABASE CONFIGURATION (Optional)
# ============================================================================
# SQLite database file paths (inside container)
# These are stored in the mounted ./data directory
# DATABASE_URL=sqlite:////app/data/scheduled_tasks.db
# SCHEDULER_DATABASE_URL=sqlite:////app/data/scheduler_jobs.db
# ============================================================================
# LOGGING CONFIGURATION (Optional)
# ============================================================================
# Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
# LOG_LEVEL=INFO
# Log file path (inside container)
# LOG_FILE=/app/logs/bdfr-web.log
# ============================================================================
# SECURITY CONFIGURATION (Production Only)
# ============================================================================
# Secret key for session management
# Generate a secure random string for production
# SECRET_KEY=your-secret-key-here
# Allowed hosts (comma-separated)
# ALLOWED_HOSTS=localhost,127.0.0.1,bdfr.example.com
# Enable CORS (true/false)
# ENABLE_CORS=true
# CORS allowed origins (comma-separated)
# CORS_ORIGINS=http://localhost:8000,https://bdfr.example.com
# ============================================================================
# NOTES
# ============================================================================
#
# - After modifying this file, restart the Docker container:
# docker-compose restart
#
# - For sensitive values in production, consider using Docker secrets:
# https://docs.docker.com/engine/swarm/secrets/
#
# - Required variables are marked as REQUIRED
# - Optional variables will use sensible defaults if not set
#
# - For more information, see DOCKER.md
+477
View File
@@ -0,0 +1,477 @@
# BDFR Web Interface - Docker Deployment Guide
This guide covers running the BDFR Web Interface with full BDFR backend support using Docker.
## Quick Start
### Prerequisites
- Docker Engine 20.10 or later
- Docker Compose 2.0 or later
- At least 2GB of free disk space
### Basic Setup
1. **Clone the repository** (if you haven't already):
```bash
git clone <repository-url>
cd BDFR_Web
```
2. **Create required directories**:
```bash
mkdir -p downloads data config
```
3. **Configure environment variables** (optional but recommended):
```bash
cp web_interface/.env.example .env
# Edit .env with your Reddit OAuth credentials
```
4. **Start the container**:
```bash
docker-compose up -d
```
5. **Access the web interface**:
Open your browser to `http://localhost:8000`
## Configuration
### Environment Variables
Create a `.env` file in the project root with the following variables:
```env
# Reddit OAuth Configuration (Required for authenticated features)
BDFR_CLIENT_ID=your_client_id_here
BDFR_CLIENT_SECRET=your_client_secret_here
BDFR_REDIRECT_URI=http://localhost:8000/auth/callback
# Server Configuration (Optional)
HOST=0.0.0.0
PORT=8000
DEBUG=false
```
### Getting Reddit OAuth Credentials
1. Go to https://www.reddit.com/prefs/apps
2. Click "Create App" or "Create Another App"
3. Fill in the form:
- **Name**: BDFR Web Interface (or your choice)
- **App type**: Select "web app"
- **Description**: (optional)
- **About URL**: (optional)
- **Redirect URI**: `http://localhost:8000/auth/callback`
4. Click "Create app"
5. Copy the **client ID** (under the app name) and **client secret**
### Volume Mounts
The Docker setup uses two volume mounts:
| Host Path | Container Path | Purpose |
|-----------|----------------|---------|
| `./downloads` | `/downloads` | All Reddit downloads are stored here |
| `./data` | `/app/data` | SQLite databases, scheduled tasks, and BDFR configuration |
**Data Directory Structure:**
```
./data/
├── bdfr-config/ # BDFR configuration and OAuth tokens
│ ├── config.cfg # BDFR settings
│ ├── log_output.txt # BDFR logs
│ └── oauth_tokens/ # Reddit authentication tokens
├── scheduled_tasks.db # Web interface scheduled tasks
└── scheduler_jobs.db # APScheduler job store
```
## Usage
### Managing the Container
**Start the container:**
```bash
docker-compose up -d
```
**Stop the container:**
```bash
docker-compose down
```
**View logs:**
```bash
docker-compose logs -f
```
**Restart the container:**
```bash
docker-compose restart
```
**Rebuild after code changes:**
```bash
docker-compose up -d --build
```
### Using the Web Interface
1. Navigate to `http://localhost:8000`
2. Use the web UI to:
- Download from subreddits
- Download from users
- Schedule recurring downloads
- Monitor active downloads
- View download history
### Using BDFR CLI Inside Container
You can also use the BDFR command-line tool directly:
**Enter the container:**
```bash
docker exec -it bdfr-web-interface bash
```
**Run BDFR commands:**
```bash
# Download from a subreddit
bdfr download /downloads --subreddit Python -L 10
# Download from a user
bdfr download /downloads --user reddituser --submitted -L 100
# Archive posts
bdfr archive /downloads --subreddit all -L 500
# Clone (download + archive)
bdfr clone /downloads --subreddit EarthPorn -L 50
```
**One-line BDFR commands:**
```bash
# Download without entering the container
docker exec bdfr-web-interface bdfr download /downloads --subreddit Python -L 10
# View BDFR version
docker exec bdfr-web-interface bdfr --version
# View BDFR help
docker exec bdfr-web-interface bdfr download --help
```
## Advanced Configuration
### Custom Port
To run on a different port, modify [`docker-compose.yml`](docker-compose.yml:11):
```yaml
ports:
- "3000:8000" # Host port 3000, container port 8000
```
Or set the PORT environment variable:
```env
PORT=3000
```
### Custom Download Directory
Mount a different host directory for downloads:
```yaml
volumes:
- /path/to/your/downloads:/downloads
- ./data:/app/data
```
### BDFR Configuration File
BDFR configuration is automatically stored in `./data/bdfr-config/`. You can customize it by creating `./data/bdfr-config/config.cfg`:
```cfg
[DEFAULT]
client_id = your_client_id
client_secret = your_client_secret
scopes = identity, read, history, mysubreddits
[bdfr]
max_wait_time = 120
time_format = %Y-%m-%d_%H-%M-%S
```
**Note:** BDFR will automatically create this directory and configuration file on first run.
### Resource Limits
Add resource constraints in [`docker-compose.yml`](docker-compose.yml):
```yaml
services:
bdfr-web:
# ... other configuration ...
deploy:
resources:
limits:
cpus: '2'
memory: 2G
reservations:
cpus: '1'
memory: 512M
```
## Troubleshooting
### Container Won't Start
**Check logs:**
```bash
docker-compose logs bdfr-web
```
**Verify ports aren't in use:**
```bash
# Windows
netstat -ano | findstr :8000
# Linux/Mac
lsof -i :8000
```
**Rebuild the image:**
```bash
docker-compose down
docker-compose build --no-cache
docker-compose up -d
```
### Permission Issues
If you encounter permission errors with volumes:
**Linux/Mac:**
```bash
sudo chown -R $USER:$USER downloads data config
chmod -R 755 downloads data config
```
**Windows:**
Ensure Docker Desktop has access to the drive where the project is located (Settings → Resources → File Sharing).
### Downloads Not Appearing
1. Check volume mounts are correct:
```bash
docker inspect bdfr-web-interface | grep -A 10 Mounts
```
2. Verify download directory inside container:
```bash
docker exec bdfr-web-interface ls -la /downloads
```
3. Check container logs for errors:
```bash
docker-compose logs -f bdfr-web
```
### Permission Denied Errors
If you see errors like "Permission denied: '/usr/local/lib/python3.11/site-packages/bdfr/default_config.cfg'":
1. Verify the BDFR config directory exists and is writable:
```bash
docker exec bdfr-web-interface ls -la /app/data/bdfr-config
```
2. Check environment variables are set correctly:
```bash
docker exec bdfr-web-interface env | grep BDFR
```
3. If the directory doesn't exist or has wrong permissions:
```bash
docker-compose down
docker-compose up -d --build
```
### OAuth Authentication Issues
1. Verify credentials in `.env` or [`docker-compose.yml`](docker-compose.yml)
2. Ensure redirect URI matches exactly: `http://localhost:8000/auth/callback`
3. Check Reddit app settings at https://www.reddit.com/prefs/apps
4. Restart container after changing credentials:
```bash
docker-compose restart
```
### High Memory Usage
BDFR can use significant memory when downloading large files or many files simultaneously:
1. Monitor memory usage:
```bash
docker stats bdfr-web-interface
```
2. Add memory limits (see Resource Limits above)
3. Reduce concurrent downloads by limiting the number of simultaneous operations
### Database Locked Errors
If you see SQLite database locked errors:
1. Ensure only one instance is running:
```bash
docker ps | grep bdfr
```
2. Stop all instances and restart:
```bash
docker-compose down
docker-compose up -d
```
## Maintenance
### Backing Up Data
**Backup downloads and database:**
```bash
# Create backup directory
mkdir -p backups
# Backup downloads
tar -czf backups/downloads-$(date +%Y%m%d).tar.gz downloads/
# Backup database
cp -r data/ backups/data-$(date +%Y%m%d)/
```
**Automated backup script** (Linux/Mac):
```bash
#!/bin/bash
# backup.sh
BACKUP_DIR="backups"
DATE=$(date +%Y%m%d_%H%M%S)
mkdir -p "$BACKUP_DIR"
tar -czf "$BACKUP_DIR/bdfr-backup-$DATE.tar.gz" downloads/ data/
# Keep only last 7 days of backups
find "$BACKUP_DIR" -name "bdfr-backup-*.tar.gz" -mtime +7 -delete
```
### Updating the Container
**Pull latest changes:**
```bash
git pull origin main
docker-compose down
docker-compose build --no-cache
docker-compose up -d
```
**Update only Docker image:**
```bash
docker-compose pull
docker-compose up -d
```
### Cleaning Up
**Remove stopped containers:**
```bash
docker-compose down
```
**Remove containers and volumes:**
```bash
docker-compose down -v
```
**Remove images:**
```bash
docker-compose down --rmi all
```
**Full cleanup (including downloads):**
```bash
docker-compose down -v --rmi all
rm -rf downloads/ data/
```
## Security Considerations
### Production Deployment
For production use, consider:
1. **Use HTTPS**: Set up a reverse proxy (nginx/Traefik) with SSL
2. **Secure credentials**: Use Docker secrets or vault for sensitive data
3. **Network isolation**: Use custom networks and restrict access
4. **Regular updates**: Keep the container and dependencies updated
5. **Monitor logs**: Set up log aggregation and monitoring
6. **Backup strategy**: Implement automated backups
### Example Nginx Reverse Proxy
```nginx
server {
listen 80;
server_name bdfr.example.com;
location / {
proxy_pass http://localhost:8000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# WebSocket support
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
}
```
## Performance Optimization
### Build Performance
**Use BuildKit for faster builds:**
```bash
DOCKER_BUILDKIT=1 docker-compose build
```
**Multi-stage build** is already implemented in the [`Dockerfile`](Dockerfile) to minimize image size.
### Runtime Performance
1. **Allocate sufficient resources** (see Resource Limits)
2. **Use SSD storage** for downloads directory
3. **Optimize network** for faster downloads
4. **Monitor container health**:
```bash
docker inspect --format='{{.State.Health.Status}}' bdfr-web-interface
```
## Support
For issues specific to:
- **Docker setup**: Check this guide and Docker logs
- **BDFR functionality**: See main [README.md](README.md)
- **Web interface**: See [web_interface/README.md](web_interface/README.md)
## Additional Resources
- [BDFR Documentation](README.md)
- [Docker Documentation](https://docs.docker.com/)
- [Docker Compose Documentation](https://docs.docker.com/compose/)
- [Reddit API Documentation](https://www.reddit.com/dev/api)
+88
View File
@@ -0,0 +1,88 @@
# Multi-stage Dockerfile for BDFR Web Interface with full BDFR backend support
# This image contains both the web interface and the complete BDFR tool
FROM python:3.11-slim AS builder
# Set working directory
WORKDIR /build
# Install build dependencies
RUN apt-get update && apt-get install -y \
gcc \
g++ \
git \
&& rm -rf /var/lib/apt/lists/*
# Copy BDFR source files and install it
COPY pyproject.toml ./
COPY bdfr/ ./bdfr/
RUN pip install --no-cache-dir .
# Copy and install web interface requirements
COPY web_interface/requirements.txt ./web_requirements.txt
RUN pip install --no-cache-dir -r web_requirements.txt
# Final stage - minimal runtime image
FROM python:3.11-slim
# Set labels
LABEL maintainer="BDFR Web Interface"
LABEL description="Complete BDFR with Web Interface - Download Reddit content via CLI or Web UI"
LABEL version="1.0.0"
# Set working directory
WORKDIR /app
# Install runtime dependencies (ffmpeg for video processing, curl for health checks)
RUN apt-get update && apt-get install -y \
ffmpeg \
curl \
&& rm -rf /var/lib/apt/lists/*
# Copy Python packages from builder (installed globally in /usr/local)
COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
COPY --from=builder /usr/local/bin /usr/local/bin
# Copy web interface files
COPY web_interface/app/ ./app/
COPY web_interface/templates/ ./templates/
COPY web_interface/static/ ./static/
COPY web_interface/start.py ./start.py
# Copy entrypoint script
COPY docker-entrypoint.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
# Create necessary directories with proper permissions
RUN mkdir -p /downloads /app/data /app/logs && \
chmod 755 /downloads /app/data /app/logs
# Create a non-root user for running the application
RUN useradd --create-home --shell /bin/bash bdfr && \
chown -R bdfr:bdfr /app /downloads
# Switch to non-root user
USER bdfr
# Set environment variables
ENV PYTHONUNBUFFERED=1
ENV BDFR_DOWNLOAD_DIR=/downloads
ENV BDFR_DATA_DIR=/app/data
ENV BDFR_CONFIG_DIR=/app/data/bdfr-config
ENV APPDATA=/app/data/bdfr-config
ENV XDG_CONFIG_HOME=/app/data/bdfr-config
# Expose port for web interface
EXPOSE 8000
# Health check - verify web interface is responding
HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \
CMD curl -f http://localhost:8000/health || exit 1
# Set entrypoint to handle runtime directory creation
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
# Default command - start web interface
# Users can override this to run BDFR CLI commands
CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
+25
View File
@@ -47,6 +47,31 @@ If on Arch Linux or derivative operating systems such as Manjaro, the BDFR can b
If you want to use the source code or make contributions, refer to [CONTRIBUTING](docs/CONTRIBUTING.md#preparing-the-environment-for-development)
### Docker Deployment
The BDFR Web Interface provides a complete Docker setup with both the CLI tool and web interface. This is the easiest way to get started:
**Quick Start (Linux/Mac):**
```bash
chmod +x docker-quick-start.sh
./docker-quick-start.sh
```
**Quick Start (Windows):**
```cmd
docker-quick-start.bat
```
**Manual Start:**
```bash
docker-compose up -d
```
The web interface will be available at `http://localhost:8000`. All downloads are stored in the `./downloads` directory on your host machine.
For detailed Docker documentation, including configuration, volume management, and troubleshooting, see [DOCKER.md](DOCKER.md).
## Usage
The BDFR works by taking submissions from a variety of "sources" from Reddit and then parsing them to download. These sources might be a subreddit, multireddit, a user list, or individual links. These sources are combined and downloaded to disk, according to a naming and organisational scheme defined by the user.
+57
View File
@@ -0,0 +1,57 @@
version: '3.8'
services:
bdfr-web:
build:
context: .
dockerfile: Dockerfile
container_name: bdfr-web-interface
ports:
- "8000:8000"
volumes:
# Downloads directory - all Reddit content goes here
- ./downloads:/downloads
# Database persistence - SQLite databases, scheduled tasks, and BDFR config
- ./data:/app/data
environment:
# Reddit OAuth Configuration
# IMPORTANT: Set these in a .env file or directly here
- BDFR_CLIENT_ID=${BDFR_CLIENT_ID:-}
- BDFR_CLIENT_SECRET=${BDFR_CLIENT_SECRET:-}
- BDFR_REDIRECT_URI=${BDFR_REDIRECT_URI:-http://localhost:8000/auth/callback}
# Server Configuration
- HOST=${HOST:-0.0.0.0}
- PORT=${PORT:-8000}
- DEBUG=${DEBUG:-false}
# Download Configuration
- BDFR_DOWNLOAD_DIR=/downloads
- BDFR_DATA_DIR=/app/data
# BDFR Configuration - stored in mounted data directory
- BDFR_CONFIG_DIR=/app/data/bdfr-config
- APPDATA=/app/data/bdfr-config
- XDG_CONFIG_HOME=/app/data/bdfr-config
# Python Configuration
- PYTHONUNBUFFERED=1
restart: unless-stopped
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
networks:
- bdfr-network
networks:
bdfr-network:
driver: bridge
# Named volumes configuration (optional)
# Uncomment to use named volumes instead of bind mounts
# volumes:
# downloads:
# data:
+10
View File
@@ -0,0 +1,10 @@
#!/bin/bash
set -e
# Create BDFR config directory if it doesn't exist
# This needs to be done at runtime because /app/data is a volume mount
mkdir -p /app/data/bdfr-config
chmod 755 /app/data/bdfr-config
# Execute the main command
exec "$@"
+132
View File
@@ -0,0 +1,132 @@
@echo off
REM Quick start script for BDFR Web Interface with Docker (Windows)
REM This script sets up and starts the Docker environment
echo ======================================
echo BDFR Web Interface - Docker Quick Start
echo ======================================
echo.
REM Check if Docker is installed
docker --version >nul 2>&1
if %errorlevel% neq 0 (
echo Error: Docker is not installed
echo Please install Docker Desktop from: https://docs.docker.com/desktop/install/windows-install/
pause
exit /b 1
)
echo Docker is installed
echo.
REM Check if Docker Compose is installed
docker-compose --version >nul 2>&1
if %errorlevel% neq 0 (
docker compose version >nul 2>&1
if %errorlevel% neq 0 (
echo Error: Docker Compose is not installed
echo Please ensure Docker Desktop is properly installed
pause
exit /b 1
)
)
echo Docker Compose is installed
echo.
REM Create required directories
echo Creating required directories...
if not exist "downloads" mkdir downloads
if not exist "data" mkdir data
if not exist "config" mkdir config
if not exist "logs" mkdir logs
echo Directories created
echo.
REM Check if .env file exists
if not exist ".env" (
echo Creating .env file from template...
if exist ".env.example" (
copy .env.example .env >nul
echo .env file created
echo.
echo IMPORTANT: Edit .env file with your Reddit OAuth credentials
echo Get credentials from: https://www.reddit.com/prefs/apps
echo.
pause
) else (
echo Warning: .env.example not found
)
) else (
echo .env file already exists
)
echo.
REM Build and start containers
echo Building Docker image...
echo This may take a few minutes on first run...
echo.
docker-compose --version >nul 2>&1
if %errorlevel% equ 0 (
docker-compose build
) else (
docker compose build
)
if %errorlevel% neq 0 (
echo.
echo Error: Failed to build Docker image
pause
exit /b 1
)
echo.
echo Build complete!
echo.
echo Starting containers...
docker-compose --version >nul 2>&1
if %errorlevel% equ 0 (
docker-compose up -d
) else (
docker compose up -d
)
if %errorlevel% neq 0 (
echo.
echo Error: Failed to start containers
pause
exit /b 1
)
echo.
echo ======================================
echo BDFR Web Interface is starting!
echo ======================================
echo.
echo Web Interface: http://localhost:8000
echo API Documentation: http://localhost:8000/docs
echo.
echo Useful commands:
echo View logs: docker-compose logs -f
echo Stop container: docker-compose down
echo Restart: docker-compose restart
echo.
echo For more information, see DOCKER.md
echo.
REM Wait a moment and check if container is running
timeout /t 3 >nul
docker ps | findstr "bdfr-web-interface" >nul 2>&1
if %errorlevel% equ 0 (
echo Container is running successfully!
) else (
echo Warning: Container may not be running. Check logs with:
echo docker-compose logs
)
echo.
pause
+110
View File
@@ -0,0 +1,110 @@
#!/bin/bash
# Quick start script for BDFR Web Interface with Docker
# This script sets up and starts the Docker environment
set -e
echo "======================================"
echo "BDFR Web Interface - Docker Quick Start"
echo "======================================"
echo ""
# Check if Docker is installed
if ! command -v docker &> /dev/null; then
echo "❌ Error: Docker is not installed"
echo "Please install Docker from: https://docs.docker.com/get-docker/"
exit 1
fi
# Check if Docker Compose is installed
if ! command -v docker-compose &> /dev/null && ! docker compose version &> /dev/null; then
echo "❌ Error: Docker Compose is not installed"
echo "Please install Docker Compose from: https://docs.docker.com/compose/install/"
exit 1
fi
echo "✅ Docker is installed"
echo ""
# Create required directories
echo "Creating required directories..."
mkdir -p downloads data config logs
echo "✅ Directories created"
echo ""
# Check if .env file exists
if [ ! -f .env ]; then
echo "📝 Creating .env file from template..."
if [ -f .env.example ]; then
cp .env.example .env
echo "✅ Created .env file"
echo ""
echo "⚠️ IMPORTANT: Edit .env file with your Reddit OAuth credentials"
echo " Get credentials from: https://www.reddit.com/prefs/apps"
echo ""
read -p "Press Enter to continue or Ctrl+C to exit and configure .env first..."
else
echo "⚠️ Warning: .env.example not found"
fi
else
echo "✅ .env file already exists"
fi
echo ""
# Build and start containers
echo "Building Docker image..."
echo "This may take a few minutes on first run..."
echo ""
if command -v docker-compose &> /dev/null; then
docker-compose build
else
docker compose build
fi
echo ""
echo "✅ Build complete!"
echo ""
echo "Starting containers..."
if command -v docker-compose &> /dev/null; then
docker-compose up -d
else
docker compose up -d
fi
echo ""
echo "======================================"
echo "✅ BDFR Web Interface is starting!"
echo "======================================"
echo ""
echo "🌐 Web Interface: http://localhost:8000"
echo "📚 API Documentation: http://localhost:8000/docs"
echo ""
echo "Useful commands:"
echo " View logs: docker-compose logs -f"
echo " Stop container: docker-compose down"
echo " Restart: docker-compose restart"
echo ""
echo "For more information, see DOCKER.md"
echo ""
# Wait a moment and check if container is running
sleep 3
if command -v docker-compose &> /dev/null; then
if docker-compose ps | grep -q "Up"; then
echo "✅ Container is running successfully!"
else
echo "⚠️ Container may not be running. Check logs with:"
echo " docker-compose logs"
fi
else
if docker compose ps | grep -q "Up"; then
echo "✅ Container is running successfully!"
else
echo "⚠️ Container may not be running. Check logs with:"
echo " docker compose logs"
fi
fi
-17
View File
@@ -1,17 +0,0 @@
# BDFR Web Interface Configuration
# Copy this file to .env and update the values as needed
# Reddit OAuth Configuration
# You MUST set this to match your Reddit OAuth app settings
# Go to https://www.reddit.com/prefs/apps, create/edit your app, and use the exact redirect URI
BDFR_REDIRECT_URI=http://localhost:8000/auth/callback
# OAuth Credentials (from your Reddit OAuth app)
# Get these from: https://www.reddit.com/prefs/apps
BDFR_CLIENT_ID=your_client_id_here
BDFR_CLIENT_SECRET=your_client_secret_here
# Server Configuration (optional)
# HOST=0.0.0.0
# PORT=8000
# DEBUG=true
+319
View File
@@ -0,0 +1,319 @@
# Scheduled Downloads Feature
## Overview
The BDFR Web Interface now supports scheduled downloads, allowing users to configure downloads that run automatically on a daily basis. This feature is designed to work seamlessly in Docker containers and ensures sequential execution to prevent system overload.
## Features
### 1. **Daily Scheduling**
- Tasks run once per day at a user-specified time
- Time is specified in the user's local timezone and automatically converted to UTC for container execution
- Automatically sets `time_filter="day"` to only download content from the last 24 hours
- Always enables `no_dupes=True` to avoid re-downloading existing content
### 2. **Sequential Execution**
- Tasks are processed one at a time through a queue system
- Manual "Run Now" tasks have priority over scheduled tasks
- Queue status is displayed in real-time
### 3. **Persistent Storage**
- SQLite database stores task configurations
- APScheduler job store ensures tasks persist across container restarts
- Execution history tracked for each task
### 4. **Full Task Management**
- Create, enable/disable, and delete scheduled tasks
- Run tasks manually on-demand
- View last run and next scheduled run times
## Architecture
### Backend Components
#### 1. **Database Models** (`web_interface/app/models.py`)
- `ScheduledTask`: Stores task configuration
- Fields: name, source (subreddit/user), schedule, timezone, etc.
- Automatically sets time_filter="day" and no_dupes=True
- `TaskExecutionHistory`: Tracks each execution
- Fields: task_id, status, items downloaded, errors, etc.
#### 2. **Task Queue** (`web_interface/app/task_queue.py`)
- `TaskQueue` class manages sequential execution
- Priority queue: 0=scheduled, 1=manual
- Blocks until each download completes before starting the next
- Thread-safe using asyncio
#### 3. **Scheduler Service** (`web_interface/app/scheduler.py`)
- APScheduler with SQLAlchemy job store for persistence
- Functions:
- `schedule_task()`: Creates cron job
- `queue_scheduled_task()`: Adds task to queue (called by scheduler)
- `execute_scheduled_task()`: Executes download and waits for completion
- `wait_for_download_completion()`: Polls every 5 seconds until done
#### 4. **API Endpoints** (`web_interface/app/scheduled_tasks.py`)
```
POST /api/scheduled-tasks - Create task
GET /api/scheduled-tasks - List all tasks
GET /api/scheduled-tasks/{id} - Get specific task
PUT /api/scheduled-tasks/{id} - Update task
DELETE /api/scheduled-tasks/{id} - Delete task
POST /api/scheduled-tasks/{id}/toggle - Enable/disable
POST /api/scheduled-tasks/{id}/run-now - Queue immediately
GET /api/scheduled-tasks/{id}/history - Execution history
GET /api/scheduled-tasks/queue/status - Queue status
```
### Frontend Components
#### 1. **HTML** (`web_interface/templates/index.html`)
- "Run Daily" checkbox in Advanced Options
- Schedule configuration fields (task name, run time)
- Scheduled Downloads section with task cards
- Queue status badge
#### 2. **JavaScript** (`web_interface/static/js/app.js`)
- `loadScheduledTasks()`: Fetches and renders tasks
- `createScheduledTask()`: Creates new scheduled task
- `toggleTask()`, `deleteTask()`, `runTaskNow()`: Task management
- `updateQueueStatus()`: Polls queue every 10 seconds
- Auto-detects browser timezone via `Intl.DateTimeFormat()`
#### 3. **CSS** (`web_interface/static/css/style.css`)
- Task card styling with hover effects
- Status badges (enabled/disabled)
- Schedule options panel
- Queue status badge
## Usage Guide
### Creating a Scheduled Download
1. **Configure Download Settings**
- Select download mode (Download/Archive/Clone)
- Choose source type (Subreddit/User)
- Enter source name
- Set limit, sort, and other options
2. **Enable Scheduling**
- Check "Run Daily" in Advanced Options
- Enter a task name (e.g., "Daily Python Posts")
- Select run time (24-hour format, in your local timezone)
3. **Submit**
- Click "Start Download" to create the scheduled task
- Task appears in the Scheduled Downloads section
- First run scheduled for the specified time
### Managing Scheduled Tasks
Each task card shows:
- Task name and source
- Download mode
- Schedule (daily at X time)
- Last run and next run times
- Status (Enabled/Disabled)
**Actions:**
- **Disable/Enable**: Toggle task on/off without deleting
- **Run Now**: Add task to queue immediately (higher priority)
- **Delete**: Remove task permanently
### Queue System
- Queue status badge shows number of tasks waiting
- Tasks execute one at a time to prevent overload
- Manual "Run Now" tasks have priority over scheduled tasks
- Download progress appears in Progress section
## Docker Deployment
### Volume Mounts Required
```yaml
volumes:
- ./downloads:/downloads # Downloaded files
- ./data:/app/data # Database and job store
```
### Database Location
- SQLite: `/app/data/scheduled_tasks.db`
- APScheduler job store: Same database
### Timezone Handling
- User specifies time in their local timezone
- Frontend auto-detects timezone via JavaScript
- Backend converts to UTC for container execution
- Cron jobs run at correct local time regardless of container timezone
## Technical Details
### Sequential Execution Flow
1. APScheduler triggers at scheduled time
2. Scheduler calls `queue_scheduled_task(task_id)`
3. Task added to queue with priority 0
4. Queue worker picks up task
5. `execute_scheduled_task()` called
6. Downloads via existing BDFR API
7. `wait_for_download_completion()` polls every 5s
8. Once complete, queue processes next task
9. Execution history recorded
### Time Filter Logic
For scheduled tasks:
- `time_filter` is automatically set to "day"
- This filters Reddit API to only return posts from last 24 hours
- Combined with daily scheduling, ensures only new content downloaded
- Prevents re-downloading old content
### Duplicate Prevention
For scheduled tasks:
- `no_dupes` is automatically enabled
- Uses existing BDFR duplicate detection
- Checks file hashes or URLs (depending on simple_check setting)
- Skips files that already exist
## Testing Checklist
### Basic Functionality
- [ ] Create scheduled task via "Run Daily" checkbox
- [ ] Task appears in Scheduled Downloads section
- [ ] Task name, source, and schedule displayed correctly
- [ ] Enable/disable toggle works
- [ ] Delete removes task
### Execution
- [ ] Manual "Run Now" triggers download immediately
- [ ] Download progress appears in Progress section
- [ ] Task completes successfully
- [ ] Execution history recorded
### Sequential Processing
- [ ] Queue multiple tasks via "Run Now"
- [ ] Tasks execute one at a time (not concurrent)
- [ ] Queue badge shows correct count
- [ ] Manual tasks execute before scheduled tasks
### Persistence
- [ ] Restart container/server
- [ ] Tasks still present after restart
- [ ] Scheduled jobs still execute at correct time
- [ ] Execution history preserved
### Timezone Handling
- [ ] Create task with different timezone
- [ ] Task runs at correct local time
- [ ] Next run time displays in user's timezone
### Edge Cases
- [ ] Create task with invalid source name
- [ ] Disable task, verify it doesn't run
- [ ] Enable disabled task
- [ ] Delete task while it's running
- [ ] Run same task multiple times quickly
## Troubleshooting
### Tasks Not Running
1. Check container logs for scheduler errors
2. Verify `/app/data` volume is mounted
3. Check database file permissions
4. Verify APScheduler is running (`scheduler.running()`)
### Queue Stuck
1. Check task_queue status in logs
2. Verify WebSocket connection for progress updates
3. Restart container to reset queue
### Timezone Issues
1. Verify browser timezone detection in DevTools
2. Check conversion in scheduler logs
3. Ensure container has correct UTC time
### Database Issues
1. Check `/app/data/scheduled_tasks.db` exists
2. Verify write permissions
3. Use SQLite browser to inspect tables
4. Check for migration errors in logs
## Future Enhancements
Potential improvements:
- [ ] Weekly scheduling option
- [ ] Custom time filters (last 3 days, last week, etc.)
- [ ] Email notifications on completion/failure
- [ ] Retry logic for failed tasks
- [ ] Task templates for quick setup
- [ ] Bulk operations (enable/disable multiple tasks)
- [ ] Advanced schedule expressions (cron syntax)
- [ ] Export/import task configurations
- [ ] Task execution statistics and charts
- [ ] Pause/resume queue
## API Examples
### Create Task
```bash
curl -X POST http://localhost:8000/api/scheduled-tasks \
-H "Content-Type: application/json" \
-d '{
"name": "Daily Python Posts",
"source_type": "subreddit",
"source_name": "python",
"download_mode": "download",
"limit": 50,
"sort": "hot",
"run_time": "02:00",
"timezone": "Pacific/Auckland",
"enabled": true
}'
```
### List Tasks
```bash
curl http://localhost:8000/api/scheduled-tasks
```
### Toggle Task
```bash
curl -X POST http://localhost:8000/api/scheduled-tasks/1/toggle
```
### Run Task Now
```bash
curl -X POST http://localhost:8000/api/scheduled-tasks/1/run-now
```
### Get Queue Status
```bash
curl http://localhost:8000/api/scheduled-tasks/queue/status
```
## Files Modified/Created
### Created:
- `web_interface/app/database.py` - Database configuration
- `web_interface/app/models.py` - ORM models
- `web_interface/app/task_queue.py` - Queue manager
- `web_interface/app/scheduler.py` - Scheduler service
- `web_interface/app/scheduled_tasks.py` - API endpoints
- `web_interface/SCHEDULED_DOWNLOADS.md` - This file
### Modified:
- `web_interface/requirements.txt` - Added dependencies
- `web_interface/app/main.py` - Integrated scheduler
- `web_interface/templates/index.html` - Added UI elements
- `web_interface/static/js/app.js` - Added JavaScript functions
- `web_interface/static/css/style.css` - Added styles
## Dependencies Added
```
sqlalchemy>=2.0.0
alembic>=1.12.0
apscheduler>=3.10.0
pytz>=2023.3
+70
View File
@@ -0,0 +1,70 @@
"""
Database configuration for scheduled downloads.
Uses SQLite with SQLAlchemy ORM for persistent storage.
"""
from sqlalchemy import create_engine, event
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from pathlib import Path
import logging
logger = logging.getLogger(__name__)
# Database file location - uses /app/data in Docker container
# This directory should be mounted as a volume for persistence
DATA_DIR = Path(__file__).parent.parent / "data"
DB_PATH = DATA_DIR / "scheduled_tasks.db"
# Ensure data directory exists (will be mounted volume in Docker)
DATA_DIR.mkdir(parents=True, exist_ok=True)
# Create engine with proper settings for SQLite in Docker
engine = create_engine(
f"sqlite:///{DB_PATH}",
echo=False,
connect_args={
"check_same_thread": False, # Allow multi-threaded access
"timeout": 30 # Longer timeout for container I/O
},
pool_pre_ping=True, # Verify connections before using
)
# Enable foreign keys for SQLite
@event.listens_for(engine, "connect")
def set_sqlite_pragma(dbapi_conn, connection_record):
cursor = dbapi_conn.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
# Create session factory
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
# Base class for models
Base = declarative_base()
def init_database():
"""Initialize database - called on container startup"""
try:
Base.metadata.create_all(bind=engine)
logger.info(f"Database initialized at {DB_PATH}")
except Exception as e:
logger.error(f"Failed to initialize database: {e}")
raise
def get_db():
"""
Dependency for FastAPI to get database session.
Usage:
@app.get("/endpoint")
def endpoint(db: Session = Depends(get_db)):
...
"""
db = SessionLocal()
try:
yield db
finally:
db.close()
+47
View File
@@ -28,6 +28,11 @@ logger = logging.getLogger(__name__)
# Import authentication module
from .auth import init_oauth_manager, get_oauth_manager
# Import scheduled tasks modules
from .database import init_database
from .scheduler import start_scheduler, stop_scheduler
from .scheduled_tasks import router as scheduled_tasks_router
# Import BDFR API layer
import sys
import os
@@ -72,6 +77,45 @@ except ImportError as e:
app = FastAPI(title="BDFR Web Interface", version="1.0.0")
# Application lifecycle events
@app.on_event("startup")
async def startup_event():
"""Initialize services on application startup"""
try:
logger.info("Starting up BDFR Web Interface...")
# Initialize database
init_database()
logger.info("Database initialized")
# Start scheduler
start_scheduler()
logger.info("Scheduler started")
logger.info("Startup complete!")
except Exception as e:
logger.error(f"Startup error: {e}", exc_info=True)
raise
@app.on_event("shutdown")
async def shutdown_event():
"""Cleanup on application shutdown"""
try:
logger.info("Shutting down BDFR Web Interface...")
# Stop scheduler
stop_scheduler()
logger.info("Scheduler stopped")
# Stop task queue
from .task_queue import task_queue
await task_queue.stop()
logger.info("Task queue stopped")
logger.info("Shutdown complete!")
except Exception as e:
logger.error(f"Shutdown error: {e}", exc_info=True)
# Initialize OAuth2 manager
def init_oauth():
"""Initialize OAuth2 manager with credentials from environment or BDFR config"""
@@ -131,6 +175,9 @@ os.makedirs(template_dir, exist_ok=True)
app.mount("/static", StaticFiles(directory=static_dir), name="static")
templates = Jinja2Templates(directory=template_dir)
# Include scheduled tasks router
app.include_router(scheduled_tasks_router)
# WebSocket connection manager
class ConnectionManager:
def __init__(self):
+130
View File
@@ -0,0 +1,130 @@
"""
Database models for scheduled downloads.
"""
from sqlalchemy import Column, String, Integer, Boolean, DateTime, Time, ForeignKey, Text
from sqlalchemy.orm import relationship
from datetime import datetime, time as time_type
import uuid
import pytz
from .database import Base
def generate_uuid():
"""Generate a UUID string"""
return str(uuid.uuid4())
class ScheduledTask(Base):
"""
Represents a scheduled download task.
Tasks run at a specified time each day.
"""
__tablename__ = "scheduled_tasks"
# Primary key
id = Column(String(36), primary_key=True, default=generate_uuid)
# Task identification
name = Column(String(255), nullable=False, index=True)
enabled = Column(Boolean, default=True, nullable=False, index=True)
# Download configuration
source_type = Column(String(20), nullable=False) # "subreddit" or "user"
source_name = Column(String(255), nullable=False, index=True)
download_mode = Column(String(20), nullable=False) # "download", "archive", "clone"
# Filter options
limit = Column(Integer, default=25, nullable=False)
sort = Column(String(20), default="hot", nullable=False)
time_filter = Column(String(20), default="day", nullable=False) # Always "day" for daily tasks
min_score = Column(Integer, nullable=True)
no_dupes = Column(Boolean, default=True, nullable=False) # Always true for scheduled
simple_check = Column(Boolean, default=False, nullable=False)
# Scheduling
schedule_frequency = Column(String(20), default="daily", nullable=False)
run_time = Column(Time, nullable=False) # Time of day to run
timezone = Column(String(50), default="UTC", nullable=False)
# Metadata
created_at = Column(DateTime, default=lambda: datetime.now(pytz.UTC), nullable=False)
updated_at = Column(DateTime, default=lambda: datetime.now(pytz.UTC), onupdate=lambda: datetime.now(pytz.UTC), nullable=False)
last_run_at = Column(DateTime, nullable=True, index=True)
next_run_at = Column(DateTime, nullable=True, index=True)
# Authentication
auth_state = Column(String(255), nullable=True)
# Relationships
executions = relationship("TaskExecutionHistory", back_populates="task", cascade="all, delete-orphan")
def to_dict(self):
"""Convert to dictionary for JSON serialization"""
return {
"id": self.id,
"name": self.name,
"enabled": self.enabled,
"source_type": self.source_type,
"source_name": self.source_name,
"download_mode": self.download_mode,
"limit": self.limit,
"sort": self.sort,
"time_filter": self.time_filter,
"min_score": self.min_score,
"no_dupes": self.no_dupes,
"simple_check": self.simple_check,
"schedule_frequency": self.schedule_frequency,
"run_time": self.run_time.isoformat() if self.run_time else None,
"timezone": self.timezone,
"created_at": self.created_at.isoformat() if self.created_at else None,
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
"last_run_at": self.last_run_at.isoformat() if self.last_run_at else None,
"next_run_at": self.next_run_at.isoformat() if self.next_run_at else None,
"auth_state": self.auth_state
}
class TaskExecutionHistory(Base):
"""
Records each execution of a scheduled task.
Tracks success/failure and metrics.
"""
__tablename__ = "task_execution_history"
# Primary key
id = Column(String(36), primary_key=True, default=generate_uuid)
# Foreign key to task
task_id = Column(String(36), ForeignKey("scheduled_tasks.id", ondelete="CASCADE"), nullable=False, index=True)
# Execution details
started_at = Column(DateTime, default=lambda: datetime.now(pytz.UTC), nullable=False, index=True)
completed_at = Column(DateTime, nullable=True)
status = Column(String(20), default="queued", nullable=False, index=True) # queued, running, success, failed, skipped
# Results
items_found = Column(Integer, default=0, nullable=False)
items_downloaded = Column(Integer, default=0, nullable=False)
error_message = Column(Text, nullable=True)
# Link to download
download_id = Column(String(255), nullable=True, index=True)
# Relationships
task = relationship("ScheduledTask", back_populates="executions")
def to_dict(self):
"""Convert to dictionary for JSON serialization"""
return {
"id": self.id,
"task_id": self.task_id,
"started_at": self.started_at.isoformat() if self.started_at else None,
"completed_at": self.completed_at.isoformat() if self.completed_at else None,
"status": self.status,
"items_found": self.items_found,
"items_downloaded": self.items_downloaded,
"error_message": self.error_message,
"download_id": self.download_id
}
+474
View File
@@ -0,0 +1,474 @@
"""
API endpoints and business logic for scheduled tasks management.
"""
from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy.orm import Session
from pydantic import BaseModel, Field
from typing import List, Optional
from datetime import datetime, time as time_type
import pytz
import logging
from .database import get_db
from .models import ScheduledTask, TaskExecutionHistory
from .scheduler import schedule_task, unschedule_task, calculate_next_run
from .task_queue import task_queue
logger = logging.getLogger(__name__)
# Create router
router = APIRouter(prefix="/api/scheduled-tasks", tags=["scheduled-tasks"])
# Pydantic models for API
class ScheduledTaskCreate(BaseModel):
name: str = Field(..., min_length=1, max_length=255)
source_type: str = Field(..., pattern="^(subreddit|user)$")
source_name: str = Field(..., min_length=1, max_length=255)
download_mode: str = Field(..., pattern="^(download|archive|clone)$")
limit: int = Field(default=25, ge=1, le=1000)
sort: str = Field(default="hot")
min_score: Optional[int] = None
simple_check: bool = False
run_time: str = Field(..., pattern="^([01]?[0-9]|2[0-3]):[0-5][0-9]$") # HH:MM format
timezone: str = Field(default="UTC")
auth_state: Optional[str] = None
class ScheduledTaskUpdate(BaseModel):
name: Optional[str] = Field(None, min_length=1, max_length=255)
limit: Optional[int] = Field(None, ge=1, le=1000)
sort: Optional[str] = None
min_score: Optional[int] = None
simple_check: Optional[bool] = None
run_time: Optional[str] = Field(None, pattern="^([01]?[0-9]|2[0-3]):[0-5][0-9]$")
timezone: Optional[str] = None
enabled: Optional[bool] = None
class ScheduledTaskResponse(BaseModel):
id: str
name: str
enabled: bool
source_type: str
source_name: str
download_mode: str
limit: int
sort: str
time_filter: str
min_score: Optional[int]
no_dupes: bool
simple_check: bool
schedule_frequency: str
run_time: str
timezone: str
created_at: str
updated_at: str
last_run_at: Optional[str]
next_run_at: Optional[str]
class Config:
from_attributes = True
class TaskExecutionResponse(BaseModel):
id: str
task_id: str
started_at: str
completed_at: Optional[str]
status: str
items_found: int
items_downloaded: int
error_message: Optional[str]
download_id: Optional[str]
class Config:
from_attributes = True
# API Endpoints
@router.post("", response_model=ScheduledTaskResponse, status_code=status.HTTP_201_CREATED)
async def create_scheduled_task(task_data: ScheduledTaskCreate, db: Session = Depends(get_db)):
"""
Create a new scheduled download task.
"""
try:
# Validate timezone
try:
pytz.timezone(task_data.timezone)
except pytz.exceptions.UnknownTimeZoneError:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Invalid timezone: {task_data.timezone}"
)
# Parse run_time
hour, minute = map(int, task_data.run_time.split(':'))
run_time_obj = time_type(hour=hour, minute=minute)
# Create task
task = ScheduledTask(
name=task_data.name,
enabled=True,
source_type=task_data.source_type,
source_name=task_data.source_name,
download_mode=task_data.download_mode,
limit=task_data.limit,
sort=task_data.sort,
time_filter="day", # Always "day" for daily scheduled tasks
min_score=task_data.min_score,
no_dupes=True, # Always true for scheduled tasks
simple_check=task_data.simple_check,
schedule_frequency="daily",
run_time=run_time_obj,
timezone=task_data.timezone,
auth_state=task_data.auth_state
)
# Calculate next run time
task.next_run_at = calculate_next_run(task)
# Save to database
db.add(task)
db.commit()
db.refresh(task)
# Schedule the task
schedule_task(task)
logger.info(f"Created scheduled task {task.id}: {task.name}")
return task.to_dict()
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to create scheduled task: {e}", exc_info=True)
db.rollback()
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to create scheduled task: {str(e)}"
)
@router.get("", response_model=List[ScheduledTaskResponse])
async def list_scheduled_tasks(db: Session = Depends(get_db)):
"""
Get all scheduled tasks.
"""
try:
tasks = db.query(ScheduledTask).order_by(ScheduledTask.created_at.desc()).all()
return [task.to_dict() for task in tasks]
except Exception as e:
logger.error(f"Failed to list scheduled tasks: {e}", exc_info=True)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to list scheduled tasks"
)
@router.get("/{task_id}", response_model=ScheduledTaskResponse)
async def get_scheduled_task(task_id: str, db: Session = Depends(get_db)):
"""
Get a specific scheduled task by ID.
"""
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
if not task:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Task {task_id} not found"
)
return task.to_dict()
@router.put("/{task_id}", response_model=ScheduledTaskResponse)
async def update_scheduled_task(
task_id: str,
task_data: ScheduledTaskUpdate,
db: Session = Depends(get_db)
):
"""
Update a scheduled task.
"""
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
if not task:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Task {task_id} not found"
)
try:
# Update fields
if task_data.name is not None:
task.name = task_data.name
if task_data.limit is not None:
task.limit = task_data.limit
if task_data.sort is not None:
task.sort = task_data.sort
if task_data.min_score is not None:
task.min_score = task_data.min_score
if task_data.simple_check is not None:
task.simple_check = task_data.simple_check
# Handle run_time update
reschedule_needed = False
if task_data.run_time is not None:
hour, minute = map(int, task_data.run_time.split(':'))
task.run_time = time_type(hour=hour, minute=minute)
reschedule_needed = True
# Handle timezone update
if task_data.timezone is not None:
try:
pytz.timezone(task_data.timezone)
task.timezone = task_data.timezone
reschedule_needed = True
except pytz.exceptions.UnknownTimeZoneError:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Invalid timezone: {task_data.timezone}"
)
# Handle enabled status
if task_data.enabled is not None and task_data.enabled != task.enabled:
task.enabled = task_data.enabled
reschedule_needed = True
# Update next_run_at if needed
if reschedule_needed:
if task.enabled:
task.next_run_at = calculate_next_run(task)
# Reschedule
unschedule_task(task_id)
schedule_task(task)
else:
# Unschedule if disabled
unschedule_task(task_id)
task.next_run_at = None
task.updated_at = datetime.now(pytz.UTC)
db.commit()
db.refresh(task)
logger.info(f"Updated scheduled task {task_id}")
return task.to_dict()
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to update task {task_id}: {e}", exc_info=True)
db.rollback()
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to update task: {str(e)}"
)
@router.delete("/{task_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_scheduled_task(task_id: str, db: Session = Depends(get_db)):
"""
Delete a scheduled task.
"""
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
if not task:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Task {task_id} not found"
)
try:
# Unschedule
unschedule_task(task_id)
# Delete from database (cascade will delete execution history)
db.delete(task)
db.commit()
logger.info(f"Deleted scheduled task {task_id}")
except Exception as e:
logger.error(f"Failed to delete task {task_id}: {e}", exc_info=True)
db.rollback()
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to delete task: {str(e)}"
)
@router.post("/{task_id}/toggle")
async def toggle_scheduled_task(task_id: str, db: Session = Depends(get_db)):
"""
Toggle a task's enabled status.
"""
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
if not task:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Task {task_id} not found"
)
try:
# Toggle enabled
task.enabled = not task.enabled
task.updated_at = datetime.now(pytz.UTC)
if task.enabled:
# Re-enable: schedule and calculate next run
task.next_run_at = calculate_next_run(task)
schedule_task(task)
message = "Task enabled"
else:
# Disable: unschedule
unschedule_task(task_id)
task.next_run_at = None
message = "Task disabled"
db.commit()
logger.info(f"Toggled task {task_id}: {message}")
return {
"message": message,
"enabled": task.enabled
}
except Exception as e:
logger.error(f"Failed to toggle task {task_id}: {e}", exc_info=True)
db.rollback()
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to toggle task: {str(e)}"
)
@router.post("/{task_id}/run-now")
async def run_task_now(task_id: str, db: Session = Depends(get_db)):
"""
Manually trigger a scheduled task to run now.
Adds it to the queue with high priority.
"""
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
if not task:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Task {task_id} not found"
)
try:
# Add to queue with priority (goes ahead of scheduled tasks)
await task_queue.add_task(task_id, priority=1)
queue_status = task_queue.get_queue_status()
logger.info(f"Manually queued task {task_id} for immediate execution")
return {
"message": f"Task queued for execution",
"task_id": task_id,
"queue_position": queue_status['queue_size'],
"currently_running": queue_status['current_task'],
"status": "queued" if queue_status['current_task'] else "starting"
}
except Exception as e:
logger.error(f"Failed to queue task {task_id}: {e}", exc_info=True)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to queue task: {str(e)}"
)
@router.get("/{task_id}/history", response_model=List[TaskExecutionResponse])
async def get_task_history(task_id: str, limit: int = 10, db: Session = Depends(get_db)):
"""
Get execution history for a specific task.
"""
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
if not task:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Task {task_id} not found"
)
try:
executions = db.query(TaskExecutionHistory)\
.filter(TaskExecutionHistory.task_id == task_id)\
.order_by(TaskExecutionHistory.started_at.desc())\
.limit(limit)\
.all()
return [execution.to_dict() for execution in executions]
except Exception as e:
logger.error(f"Failed to get history for task {task_id}: {e}", exc_info=True)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to get task history"
)
@router.get("/history/recent", response_model=List[TaskExecutionResponse])
async def get_recent_history(limit: int = 20, db: Session = Depends(get_db)):
"""
Get recent execution history across all tasks.
"""
try:
executions = db.query(TaskExecutionHistory)\
.order_by(TaskExecutionHistory.started_at.desc())\
.limit(limit)\
.all()
return [execution.to_dict() for execution in executions]
except Exception as e:
logger.error(f"Failed to get recent history: {e}", exc_info=True)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to get execution history"
)
@router.get("/queue/status")
async def get_queue_status():
"""Get current task queue status"""
try:
status = task_queue.get_queue_status()
# Get details of current task if any
current_task_info = None
if status['current_task']:
db = next(get_db())
try:
task = db.query(ScheduledTask).filter(ScheduledTask.id == status['current_task']).first()
if task:
current_task_info = {
'id': task.id,
'name': task.name,
'source': f"{task.source_type}/{task.source_name}"
}
finally:
db.close()
return {
'queue_size': status['queue_size'],
'is_processing': status['is_processing'],
'current_task': current_task_info
}
except Exception as e:
logger.error(f"Failed to get queue status: {e}", exc_info=True)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to get queue status"
)
+360
View File
@@ -0,0 +1,360 @@
"""
Scheduler service for managing scheduled download tasks.
Uses APScheduler with task queue for sequential execution.
"""
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.cron import CronTrigger
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
from datetime import datetime, time as time_type, timedelta
from sqlalchemy.orm import Session
import pytz
import logging
import asyncio
from .database import SessionLocal, DATA_DIR
from .models import ScheduledTask, TaskExecutionHistory
from .task_queue import task_queue
logger = logging.getLogger(__name__)
# Scheduler job store database (separate from main DB)
SCHEDULER_DB_PATH = DATA_DIR / "scheduler_jobs.db"
# Configure job stores
jobstores = {
'default': SQLAlchemyJobStore(url=f'sqlite:///{SCHEDULER_DB_PATH}')
}
# Configure scheduler for Docker container
scheduler = AsyncIOScheduler(
jobstores=jobstores,
timezone=pytz.UTC, # Container runs in UTC
job_defaults={
'coalesce': True, # Combine multiple missed executions into one
'max_instances': 1, # Only one instance of each job at a time
'misfire_grace_time': 3600 # Allow up to 1 hour late execution
}
)
async def queue_scheduled_task(task_id: str):
"""
Called by scheduler at the configured time.
Adds task to queue rather than executing immediately.
Args:
task_id: UUID of the scheduled task
"""
logger.info(f"Scheduler triggered for task {task_id}, adding to queue")
try:
await task_queue.add_task(task_id, priority=0) # Normal priority for scheduled tasks
except Exception as e:
logger.error(f"Failed to queue scheduled task {task_id}: {e}", exc_info=True)
def schedule_task(task: ScheduledTask):
"""
Schedule a task to be added to the queue at specified time.
Args:
task: ScheduledTask model instance
"""
if not task.enabled:
logger.info(f"Skipping scheduling for disabled task {task.id}")
return
try:
# Convert user's timezone to UTC for container execution
user_tz = pytz.timezone(task.timezone)
# Parse time
if isinstance(task.run_time, str):
hour, minute = map(int, task.run_time.split(':')[:2])
else:
hour = task.run_time.hour
minute = task.run_time.minute
# Create cron trigger with user's timezone
trigger = CronTrigger(
hour=hour,
minute=minute,
timezone=user_tz
)
# Add job to scheduler
scheduler.add_job(
func=queue_scheduled_task,
trigger=trigger,
args=[task.id],
id=str(task.id),
replace_existing=True,
name=f"{task.name} ({task.source_type}/{task.source_name})"
)
logger.info(f"Scheduled task {task.id} '{task.name}' for {hour:02d}:{minute:02d} {task.timezone}")
# Update next_run_at
next_run = trigger.get_next_fire_time(None, datetime.now(user_tz))
if next_run:
db = SessionLocal()
try:
db_task = db.query(ScheduledTask).filter(ScheduledTask.id == task.id).first()
if db_task:
db_task.next_run_at = next_run
db.commit()
finally:
db.close()
except Exception as e:
logger.error(f"Failed to schedule task {task.id}: {e}", exc_info=True)
def unschedule_task(task_id: str):
"""
Remove a task from the scheduler.
Args:
task_id: UUID of the scheduled task
"""
try:
if scheduler.get_job(task_id):
scheduler.remove_job(task_id)
logger.info(f"Unscheduled task {task_id}")
except Exception as e:
logger.error(f"Failed to unschedule task {task_id}: {e}", exc_info=True)
def load_scheduled_tasks():
"""
Load all enabled scheduled tasks from database and schedule them.
Called on application startup.
"""
db = SessionLocal()
try:
tasks = db.query(ScheduledTask).filter(ScheduledTask.enabled == True).all()
logger.info(f"Loading {len(tasks)} enabled scheduled tasks")
for task in tasks:
schedule_task(task)
logger.info(f"Loaded {len(tasks)} scheduled tasks")
except Exception as e:
logger.error(f"Failed to load scheduled tasks: {e}", exc_info=True)
finally:
db.close()
async def execute_scheduled_task(task_id: str):
"""
Execute a scheduled download task.
This function BLOCKS until the download is complete,
ensuring sequential execution.
Args:
task_id: UUID of the scheduled task
"""
db = SessionLocal()
try:
# Load task from database
task = db.query(ScheduledTask).filter(ScheduledTask.id == task_id).first()
if not task:
logger.error(f"Task {task_id} not found in database")
return
if not task.enabled:
logger.info(f"Skipping disabled task {task_id}")
# Still record in history that it was skipped
execution = TaskExecutionHistory(
task_id=task_id,
status='skipped',
completed_at=datetime.now(pytz.UTC)
)
db.add(execution)
db.commit()
return
logger.info(f"Executing scheduled task {task_id}: {task.source_type}/{task.source_name}")
# Create execution history record
execution = TaskExecutionHistory(
task_id=task_id,
status='running',
started_at=datetime.now(pytz.UTC)
)
db.add(execution)
db.commit()
execution_id = execution.id
try:
# Import here to avoid circular imports
from .main import create_download_with_bdfr_api, bdfr_manager
# Build download parameters
kwargs = {
'limit': task.limit,
'sort': task.sort,
'time_filter': 'day', # Always "day" for daily scheduled tasks
'no_dupes': True, # Always enabled for scheduled tasks
'simple_check': task.simple_check,
'auth_state': task.auth_state
}
# Create download using existing API
download_id = await create_download_with_bdfr_api(
download_type=task.source_type,
name=task.source_name,
**kwargs
)
logger.info(f"Task {task_id} started as download {download_id}")
# Update execution record with download_id
execution = db.query(TaskExecutionHistory).filter(TaskExecutionHistory.id == execution_id).first()
if execution:
execution.download_id = download_id
db.commit()
# **CRITICAL: Wait for download to complete before returning**
await wait_for_download_completion(download_id, bdfr_manager)
# Check final status
download_status = bdfr_manager.get_download_status(download_id)
# Update execution history
execution = db.query(TaskExecutionHistory).filter(TaskExecutionHistory.id == execution_id).first()
if execution:
if download_status and download_status['status'] == 'completed':
execution.status = 'success'
execution.items_found = download_status.get('items_found', 0)
execution.items_downloaded = download_status.get('items_processed', 0)
logger.info(f"Task {task_id} completed successfully")
else:
execution.status = 'failed'
execution.error_message = download_status.get('error', 'Unknown error') if download_status else 'Download status not found'
logger.error(f"Task {task_id} failed: {execution.error_message}")
execution.completed_at = datetime.now(pytz.UTC)
db.commit()
# Update task timestamps
task.last_run_at = datetime.now(pytz.UTC)
task.next_run_at = calculate_next_run(task)
db.commit()
except Exception as e:
logger.error(f"Task {task_id} execution error: {e}", exc_info=True)
# Update execution history with error
execution = db.query(TaskExecutionHistory).filter(TaskExecutionHistory.id == execution_id).first()
if execution:
execution.status = 'failed'
execution.error_message = str(e)
execution.completed_at = datetime.now(pytz.UTC)
db.commit()
raise
finally:
db.close()
async def wait_for_download_completion(download_id: str, bdfr_manager, timeout: int = 3600):
"""
Wait for a download to complete.
Polls the download status until it's no longer running.
Args:
download_id: The download to wait for
bdfr_manager: BDFRManager instance
timeout: Maximum seconds to wait (default 1 hour)
"""
start_time = datetime.now()
check_interval = 5 # Check every 5 seconds
logger.info(f"Waiting for download {download_id} to complete...")
while True:
# Check if timeout exceeded
elapsed = (datetime.now() - start_time).total_seconds()
if elapsed > timeout:
logger.error(f"Download {download_id} timed out after {timeout}s")
raise TimeoutError(f"Download exceeded timeout of {timeout}s")
# Check download status
status = bdfr_manager.get_download_status(download_id)
if not status:
logger.warning(f"Download {download_id} status not found, assuming complete")
break
download_status = status.get('status', 'unknown')
# Check if download is finished (completed, failed, or cancelled)
if download_status in ['completed', 'failed', 'cancelled']:
logger.info(f"Download {download_id} finished with status: {download_status}")
break
# Still running, wait before checking again
await asyncio.sleep(check_interval)
def calculate_next_run(task: ScheduledTask) -> datetime:
"""
Calculate the next run time for a task based on its schedule.
Args:
task: ScheduledTask instance
Returns:
Next run datetime in UTC
"""
user_tz = pytz.timezone(task.timezone)
now = datetime.now(user_tz)
# Parse run time
if isinstance(task.run_time, str):
hour, minute = map(int, task.run_time.split(':')[:2])
else:
hour = task.run_time.hour
minute = task.run_time.minute
# Calculate next run
next_run = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
# If time has passed today, schedule for tomorrow
if next_run <= now:
next_run += timedelta(days=1)
# Convert to UTC
return next_run.astimezone(pytz.UTC)
def start_scheduler():
"""Start the scheduler. Called on application startup."""
try:
if not scheduler.running:
scheduler.start()
logger.info("Scheduler started")
# Set execute callback for task queue
task_queue.set_execute_callback(execute_scheduled_task)
# Load existing tasks
load_scheduled_tasks()
else:
logger.info("Scheduler already running")
except Exception as e:
logger.error(f"Failed to start scheduler: {e}", exc_info=True)
raise
def stop_scheduler():
"""Stop the scheduler. Called on application shutdown."""
try:
if scheduler.running:
scheduler.shutdown(wait=True)
logger.info("Scheduler stopped")
except Exception as e:
logger.error(f"Failed to stop scheduler: {e}", exc_info=True)
+148
View File
@@ -0,0 +1,148 @@
"""
Task Queue Manager for sequential execution of scheduled downloads.
Ensures only one task runs at a time.
"""
import asyncio
from typing import Optional, Dict
from datetime import datetime
import logging
logger = logging.getLogger(__name__)
class TaskQueue:
"""
Manages sequential execution of scheduled download tasks.
Ensures only one task runs at a time.
"""
def __init__(self):
self.queue: asyncio.Queue = asyncio.Queue()
self.current_task: Optional[str] = None # Current task_id being executed
self.is_processing: bool = False
self.worker_task: Optional[asyncio.Task] = None
self._execute_callback = None
def set_execute_callback(self, callback):
"""
Set the callback function to execute tasks.
Args:
callback: Async function that takes task_id and executes it
"""
self._execute_callback = callback
async def add_task(self, task_id: str, priority: int = 0):
"""
Add a task to the queue.
Args:
task_id: UUID of the scheduled task
priority: 0 = scheduled (normal), 1 = manual "Run Now" (higher priority)
"""
task_info = {
'task_id': task_id,
'priority': priority,
'queued_at': datetime.now()
}
# For priority tasks, we need to reorder the queue
if priority > 0:
# Get all items from queue
items = []
while not self.queue.empty():
try:
items.append(await asyncio.wait_for(self.queue.get(), timeout=0.1))
except asyncio.TimeoutError:
break
# Add new priority task
await self.queue.put(task_info)
# Re-add other items
for item in items:
await self.queue.put(item)
logger.info(f"Priority task {task_id} added to front of queue (queue_size={self.queue.qsize()})")
else:
await self.queue.put(task_info)
logger.info(f"Task {task_id} added to queue (priority={priority}, queue_size={self.queue.qsize()})")
# Start worker if not already running
if not self.is_processing:
await self.start_worker()
async def start_worker(self):
"""Start the queue worker if not already running"""
if self.worker_task is None or self.worker_task.done():
self.worker_task = asyncio.create_task(self._process_queue())
logger.info("Queue worker started")
async def _process_queue(self):
"""Process tasks from queue sequentially"""
self.is_processing = True
logger.info("Queue worker processing started")
while True:
try:
# Wait for next task (with timeout to allow graceful shutdown)
try:
task_info = await asyncio.wait_for(
self.queue.get(),
timeout=60.0
)
except asyncio.TimeoutError:
# Check if queue is empty
if self.queue.empty():
logger.info("Queue empty, worker stopping")
break
continue
task_id = task_info['task_id']
self.current_task = task_id
logger.info(f"Executing task {task_id} from queue (queue_size={self.queue.qsize()})")
# Execute the task (this will block until download completes)
try:
if self._execute_callback:
await self._execute_callback(task_id)
logger.info(f"Task {task_id} completed successfully")
else:
logger.error(f"No execute callback set, cannot run task {task_id}")
except Exception as e:
logger.error(f"Task {task_id} failed: {e}", exc_info=True)
finally:
self.current_task = None
self.queue.task_done()
except Exception as e:
logger.error(f"Queue worker error: {e}", exc_info=True)
self.is_processing = False
logger.info("Queue worker stopped")
def get_queue_status(self) -> Dict:
"""Get current queue status"""
return {
'current_task': self.current_task,
'queue_size': self.queue.qsize(),
'is_processing': self.is_processing
}
async def stop(self):
"""Stop the queue worker gracefully"""
logger.info("Stopping queue worker...")
if self.worker_task and not self.worker_task.done():
# Wait for current task to complete
try:
await asyncio.wait_for(self.worker_task, timeout=300) # 5 minute timeout
except asyncio.TimeoutError:
logger.warning("Queue worker did not stop within timeout, cancelling")
self.worker_task.cancel()
logger.info("Queue worker stopped")
# Global queue instance
task_queue = TaskQueue()
Binary file not shown.
Binary file not shown.
+5 -1
View File
@@ -8,4 +8,8 @@ python-dotenv>=1.0.0
pydantic>=2.0.0
pydantic-settings>=2.0.0
requests>=2.25.0
httpx>=0.24.0
httpx>=0.24.0
sqlalchemy>=2.0.0
alembic>=1.12.0
apscheduler>=3.10.0
pytz>=2023.3
+241
View File
@@ -203,6 +203,14 @@ main {
border-top: 1px solid #e9ecef;
}
.schedule-options {
margin-top: 20px;
padding: 20px;
background: white;
border-radius: 8px;
border: 2px solid #667eea;
}
.form-section h4 {
color: #2c3e50;
font-size: 1.1rem;
@@ -770,6 +778,239 @@ footer {
opacity: 0.8;
}
/* Scheduled Downloads Section */
.scheduled-section {
margin-bottom: 40px;
}
.scheduled-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 20px;
}
.scheduled-header h2 {
color: #2c3e50;
font-size: 1.8rem;
}
.queue-status {
display: flex;
align-items: center;
gap: 10px;
}
.queue-badge {
background: rgba(102, 126, 234, 0.1);
color: #667eea;
padding: 8px 16px;
border-radius: 20px;
font-size: 0.9rem;
font-weight: 600;
border: 2px solid rgba(102, 126, 234, 0.3);
}
.scheduled-container {
background: #f8f9fa;
border-radius: 10px;
padding: 20px;
min-height: 100px;
display: flex;
align-items: center;
justify-content: center;
}
.no-tasks {
text-align: center;
color: #666;
}
.scheduled-list {
background: #f8f9fa;
border-radius: 10px;
padding: 20px;
}
.scheduled-items {
display: flex;
flex-direction: column;
gap: 15px;
}
/* Scheduled Task Card */
.task-card {
background: white;
padding: 20px;
border-radius: 8px;
border-left: 4px solid #667eea;
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
transition: all 0.3s ease;
}
.task-card:hover {
box-shadow: 0 4px 12px rgba(0,0,0,0.15);
transform: translateY(-2px);
}
.task-card.disabled {
opacity: 0.6;
border-left-color: #6c757d;
}
.task-header {
display: flex;
justify-content: space-between;
align-items: flex-start;
margin-bottom: 15px;
}
.task-info h4 {
color: #2c3e50;
margin-bottom: 8px;
font-size: 1.1rem;
}
.task-meta {
display: flex;
flex-direction: column;
gap: 4px;
font-size: 0.9rem;
color: #666;
}
.task-meta-item {
display: flex;
align-items: center;
gap: 6px;
}
.task-meta-item strong {
color: #555;
min-width: 80px;
}
.task-status {
display: flex;
align-items: center;
gap: 8px;
}
.status-badge {
font-weight: 600;
padding: 6px 12px;
border-radius: 20px;
font-size: 0.85rem;
white-space: nowrap;
}
.status-badge.enabled {
background: #d4edda;
color: #155724;
border: 1px solid #c3e6cb;
}
.status-badge.disabled {
background: #f8d7da;
color: #721c24;
border: 1px solid #f5c6cb;
}
.task-controls {
display: flex;
gap: 8px;
margin-top: 15px;
padding-top: 15px;
border-top: 1px solid #e9ecef;
}
.btn-task {
padding: 8px 16px;
border: none;
border-radius: 6px;
font-size: 0.85rem;
font-weight: 600;
cursor: pointer;
transition: all 0.2s ease;
text-transform: uppercase;
letter-spacing: 0.5px;
}
.btn-toggle {
background: #667eea;
color: white;
}
.btn-toggle:hover {
background: #5568d3;
transform: translateY(-1px);
box-shadow: 0 2px 8px rgba(102, 126, 234, 0.3);
}
.btn-toggle.disabled {
background: #28a745;
}
.btn-toggle.disabled:hover {
background: #218838;
}
.btn-run {
background: #ffc107;
color: #000;
}
.btn-run:hover {
background: #ffca2c;
transform: translateY(-1px);
box-shadow: 0 2px 8px rgba(255, 193, 7, 0.3);
}
.btn-delete {
background: #dc3545;
color: white;
}
.btn-delete:hover {
background: #c82333;
transform: translateY(-1px);
box-shadow: 0 2px 8px rgba(220, 53, 69, 0.3);
}
.btn-history {
background: #17a2b8;
color: white;
}
.btn-history:hover {
background: #138496;
transform: translateY(-1px);
box-shadow: 0 2px 8px rgba(23, 162, 184, 0.3);
}
/* Task Schedule Info */
.task-schedule {
background: rgba(102, 126, 234, 0.05);
padding: 12px;
border-radius: 6px;
margin-bottom: 10px;
border-left: 3px solid #667eea;
}
.task-schedule-item {
font-size: 0.9rem;
color: #555;
margin-bottom: 4px;
}
.task-schedule-item:last-child {
margin-bottom: 0;
}
.task-schedule-item strong {
color: #2c3e50;
}
/* Animations */
@keyframes pulse {
0% { box-shadow: 0 2px 8px rgba(0,0,0,0.1); }
+309
View File
@@ -7,6 +7,7 @@ class BDFRApp {
this.maxReconnectAttempts = 5;
this.reconnectDelay = 1000;
this.downloads = new Map();
this.scheduledTasks = new Map();
this.authState = null;
this.authenticated = false;
@@ -19,6 +20,10 @@ class BDFRApp {
// Check for stored auth state first
this.authState = this.getStoredAuthState();
this.checkAuthentication();
// Load scheduled tasks
this.loadScheduledTasks();
this.startQueuePolling();
}
initializeElements() {
@@ -32,6 +37,13 @@ class BDFRApp {
this.downloadsList = document.getElementById('downloadsList');
this.downloadsItems = document.getElementById('downloadsItems');
// Scheduled tasks containers
this.scheduledContainer = document.getElementById('scheduledContainer');
this.scheduledList = document.getElementById('scheduledList');
this.scheduledItems = document.getElementById('scheduledItems');
this.queueStatus = document.getElementById('queueStatus');
this.queueCount = document.getElementById('queueCount');
// Status elements
this.wsStatus = document.getElementById('wsStatus');
this.bdfrStatus = document.getElementById('bdfrStatus');
@@ -44,6 +56,12 @@ class BDFRApp {
this.logoutBtn = document.getElementById('logoutBtn');
this.authStateInput = document.getElementById('authState');
this.userAuthStateInput = document.getElementById('userAuthState');
// Scheduled task form elements
this.runDailyCheckbox = document.getElementById('runDaily');
this.scheduleOptions = document.getElementById('scheduleOptions');
this.taskNameInput = document.getElementById('taskName');
this.runTimeInput = document.getElementById('runTime');
}
bindEvents() {
@@ -57,6 +75,21 @@ class BDFRApp {
radio.addEventListener('change', (e) => this.updateSourceTypeUI(e.target.value));
});
}
// Run Daily checkbox toggle
if (this.runDailyCheckbox) {
this.runDailyCheckbox.addEventListener('change', (e) => {
this.scheduleOptions.style.display = e.target.checked ? 'block' : 'none';
if (e.target.checked && !this.taskNameInput.value) {
// Auto-generate task name
const sourceType = document.querySelector('input[name="source_type"]:checked').value;
const sourceName = document.getElementById('sourceName').value.trim();
if (sourceName) {
this.taskNameInput.value = `Daily ${sourceName} ${sourceType}`;
}
}
});
}
if (this.subredditForm) {
this.subredditForm.addEventListener('submit', (e) => this.handleSubredditSubmit(e));
@@ -292,6 +325,7 @@ class BDFRApp {
const sort = formData.get('sort');
const noDupes = document.getElementById('noDupes').checked;
const simpleCheck = document.getElementById('simpleCheck').checked;
const runDaily = this.runDailyCheckbox.checked;
// Validate source name
if (sourceType === 'subreddit' && !this.validateSubreddit(sourceName)) {
@@ -303,6 +337,12 @@ class BDFRApp {
return;
}
// If Run Daily is checked, create scheduled task instead
if (runDaily) {
await this.createScheduledTask(e);
return;
}
// Build confirmation message
const modeLabels = {
'download': 'Download (media files)',
@@ -994,6 +1034,275 @@ class BDFRApp {
this.showError('Authentication failed');
}
}
// ===== Scheduled Tasks Methods =====
async loadScheduledTasks() {
try {
const response = await fetch('/api/scheduled-tasks');
const tasks = await response.json();
if (response.ok) {
this.scheduledTasks.clear();
tasks.forEach(task => {
this.scheduledTasks.set(task.id, task);
});
this.renderScheduledTasks();
}
} catch (error) {
console.error('Failed to load scheduled tasks:', error);
}
}
renderScheduledTasks() {
if (this.scheduledTasks.size > 0) {
this.scheduledContainer.style.display = 'none';
this.scheduledList.style.display = 'block';
this.scheduledItems.innerHTML = '';
this.scheduledTasks.forEach(task => {
const card = this.createTaskCard(task);
this.scheduledItems.appendChild(card);
});
} else {
this.scheduledContainer.style.display = 'flex';
this.scheduledList.style.display = 'none';
}
}
createTaskCard(task) {
const card = document.createElement('div');
card.className = `task-card ${task.enabled ? '' : 'disabled'}`;
card.id = `task-${task.id}`;
const sourceLabel = task.source_type === 'subreddit' ? `r/${task.source_name}` : `u/${task.source_name}`;
const modeLabel = task.download_mode.charAt(0).toUpperCase() + task.download_mode.slice(1);
const lastRun = task.last_run_at ? new Date(task.last_run_at).toLocaleString() : 'Never';
const nextRun = task.next_run_at ? new Date(task.next_run_at).toLocaleString() : 'Not scheduled';
card.innerHTML = `
<div class="task-header">
<div class="task-info">
<h4>${task.name}</h4>
<div class="task-meta">
<div class="task-meta-item">
<strong>Source:</strong> ${sourceLabel}
</div>
<div class="task-meta-item">
<strong>Mode:</strong> ${modeLabel}
</div>
<div class="task-meta-item">
<strong>Schedule:</strong> Daily at ${task.run_time}
</div>
</div>
</div>
<div class="task-status">
<span class="status-badge ${task.enabled ? 'enabled' : 'disabled'}">
${task.enabled ? '✓ Enabled' : '✗ Disabled'}
</span>
</div>
</div>
<div class="task-schedule">
<div class="task-schedule-item"><strong>Last Run:</strong> ${lastRun}</div>
<div class="task-schedule-item"><strong>Next Run:</strong> ${nextRun}</div>
</div>
<div class="task-controls">
<button class="btn-task btn-toggle ${task.enabled ? '' : 'disabled'}" data-task-id="${task.id}" data-action="toggle">
${task.enabled ? 'Disable' : 'Enable'}
</button>
<button class="btn-task btn-run" data-task-id="${task.id}" data-action="run">
Run Now
</button>
<button class="btn-task btn-delete" data-task-id="${task.id}" data-action="delete">
Delete
</button>
</div>
`;
// Add event listeners to buttons
const toggleBtn = card.querySelector('[data-action="toggle"]');
const runBtn = card.querySelector('[data-action="run"]');
const deleteBtn = card.querySelector('[data-action="delete"]');
if (toggleBtn) {
toggleBtn.addEventListener('click', () => this.toggleTask(task.id));
}
if (runBtn) {
runBtn.addEventListener('click', () => this.runTaskNow(task.id));
}
if (deleteBtn) {
deleteBtn.addEventListener('click', () => this.deleteTask(task.id));
}
return card;
}
async createScheduledTask(e) {
const formData = new FormData(e.target);
const taskName = this.taskNameInput.value.trim();
const runTime = this.runTimeInput.value;
if (!taskName) {
this.showError('Please enter a task name');
return;
}
if (!runTime) {
this.showError('Please select a run time');
return;
}
const downloadMode = formData.get('download_mode');
const sourceType = formData.get('source_type');
const sourceName = formData.get('source_name').trim();
const limit = formData.get('limit');
const sort = formData.get('sort');
// Get browser timezone
const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
const taskData = {
name: taskName,
source_type: sourceType,
source_name: sourceName,
download_mode: downloadMode,
limit: parseInt(limit),
sort: sort,
run_time: runTime,
timezone: timezone,
enabled: true
};
try {
this.showLoading(e.target.querySelector('button'));
const response = await fetch('/api/scheduled-tasks', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(taskData)
});
const result = await response.json();
if (response.ok) {
this.showSuccess(`Scheduled task created: ${taskName}`);
e.target.reset();
this.runDailyCheckbox.checked = false;
this.scheduleOptions.style.display = 'none';
await this.loadScheduledTasks();
} else {
this.showError(result.detail || 'Failed to create scheduled task');
}
} catch (error) {
console.error('Error:', error);
this.showError('Network error occurred');
} finally {
this.hideLoading(e.target.querySelector('button'));
}
}
async toggleTask(taskId) {
try {
const response = await fetch(`/api/scheduled-tasks/${taskId}/toggle`, {
method: 'POST'
});
const result = await response.json();
if (response.ok) {
this.showSuccess(`Task ${result.enabled ? 'enabled' : 'disabled'}`);
await this.loadScheduledTasks();
} else {
this.showError(result.detail || 'Failed to toggle task');
}
} catch (error) {
console.error('Error:', error);
this.showError('Failed to toggle task');
}
}
async runTaskNow(taskId) {
if (!confirm('Run this scheduled task now?')) {
return;
}
try {
const response = await fetch(`/api/scheduled-tasks/${taskId}/run-now`, {
method: 'POST'
});
const result = await response.json();
if (response.ok) {
this.showSuccess('Task added to queue');
await this.loadScheduledTasks();
} else {
this.showError(result.detail || 'Failed to queue task');
}
} catch (error) {
console.error('Error:', error);
this.showError('Failed to queue task');
}
}
async deleteTask(taskId) {
if (!confirm('Delete this scheduled task? This cannot be undone.')) {
return;
}
try {
const response = await fetch(`/api/scheduled-tasks/${taskId}`, {
method: 'DELETE'
});
if (response.ok) {
this.showSuccess('Task deleted');
await this.loadScheduledTasks();
} else {
const result = await response.json();
this.showError(result.detail || 'Failed to delete task');
}
} catch (error) {
console.error('Error:', error);
this.showError('Failed to delete task');
}
}
async updateQueueStatus() {
try {
const response = await fetch('/api/scheduled-tasks/queue/status');
const status = await response.json();
if (response.ok) {
const queueSize = status.queue_size || 0;
if (queueSize > 0 || status.current_task) {
this.queueStatus.style.display = 'block';
this.queueCount.textContent = queueSize;
} else {
this.queueStatus.style.display = 'none';
}
}
} catch (error) {
console.error('Failed to update queue status:', error);
}
}
startQueuePolling() {
// Update queue status every 10 seconds
setInterval(() => {
this.updateQueueStatus();
}, 10000);
// Initial update
this.updateQueueStatus();
}
}
// Initialize the application when DOM is loaded
+44
View File
@@ -151,6 +151,26 @@
<span class="checkmark"></span>
Create Hard Links
</label>
<label class="checkbox-label">
<input type="checkbox" id="runDaily" name="run_daily">
<span class="checkmark"></span>
Run Daily (Scheduled Download)
</label>
</div>
<!-- Scheduled Download Options (shown when Run Daily is checked) -->
<div id="scheduleOptions" class="schedule-options" style="display: none;">
<div class="form-group">
<label for="taskName">Task Name:</label>
<input type="text" id="taskName" name="task_name" placeholder="e.g., Daily Python Posts">
<small class="form-help">A friendly name to identify this scheduled task</small>
</div>
<div class="form-group">
<label for="runTime">Run Time (24-hour format):</label>
<input type="time" id="runTime" name="run_time" value="02:00">
<small class="form-help">Time to run the download daily (in your local timezone)</small>
</div>
</div>
</div>
@@ -183,6 +203,30 @@
</div>
</section>
<!-- Scheduled Downloads Section -->
<section class="scheduled-section">
<div class="scheduled-header">
<h2>📅 Scheduled Downloads</h2>
<div class="queue-status" id="queueStatus" style="display: none;">
<span class="queue-badge">⏳ Queue: <span id="queueCount">0</span></span>
</div>
</div>
<div id="scheduledContainer" class="scheduled-container">
<div class="no-tasks">
<div class="empty-state">
<div class="empty-icon">📅</div>
<p>No scheduled downloads</p>
<p>Check "Run Daily" above to schedule automatic downloads.</p>
</div>
</div>
</div>
<!-- Scheduled Tasks List -->
<div id="scheduledList" class="scheduled-list" style="display: none;">
<div id="scheduledItems" class="scheduled-items"></div>
</div>
</section>
<!-- Status Section -->
<section class="status-section">
<div class="status-card">