fixed file naming thingy

This commit is contained in:
2025-11-06 21:53:32 +13:00
parent 6d9a078656
commit 8f8e2c744d
9 changed files with 541 additions and 15 deletions
+89
View File
@@ -47,13 +47,102 @@ Base = declarative_base()
def init_database():
"""Initialize database - called on container startup"""
try:
# Check if database file exists
if DB_PATH.exists():
logger.info(f"Database file exists at {DB_PATH}, checking for schema upgrades")
upgrade_database_schema()
else:
logger.info(f"Creating new database at {DB_PATH}")
Base.metadata.create_all(bind=engine)
logger.info(f"Database initialized at {DB_PATH}")
# Verify the schema after creation/upgrade
verify_database_schema()
except Exception as e:
logger.error(f"Failed to initialize database: {e}")
raise
def verify_database_schema():
"""Verify that the database schema has all required columns"""
try:
from sqlalchemy import text
with engine.connect() as conn:
result = conn.execute(text("PRAGMA table_info(scheduled_tasks)"))
columns = [row[1] for row in result.fetchall()]
required_columns = ['upvoted', 'saved']
missing_columns = [col for col in required_columns if col not in columns]
if missing_columns:
logger.error(f"Database schema verification failed. Missing columns: {missing_columns}")
logger.error(f"Available columns: {columns}")
else:
logger.info("Database schema verification passed")
except Exception as e:
logger.error(f"Failed to verify database schema: {e}")
def upgrade_database_schema():
"""Check and upgrade database schema for new columns"""
try:
from sqlalchemy import text
logger.info("Starting database schema upgrade check...")
# Check if scheduled_tasks table exists first
with engine.connect() as conn:
# Check if table exists
result = conn.execute(text("SELECT name FROM sqlite_master WHERE type='table' AND name='scheduled_tasks'"))
table_exists = result.fetchone() is not None
if not table_exists:
logger.info("scheduled_tasks table does not exist, will be created by create_all")
return
logger.info("scheduled_tasks table exists, checking columns...")
# Get table info
result = conn.execute(text("PRAGMA table_info(scheduled_tasks)"))
columns = [row[1] for row in result.fetchall()]
logger.info(f"Existing columns in scheduled_tasks: {columns}")
# Check for missing columns
missing_columns = []
if 'upvoted' not in columns:
missing_columns.append('upvoted BOOLEAN DEFAULT 0')
if 'saved' not in columns:
missing_columns.append('saved BOOLEAN DEFAULT 0')
# Add missing columns
for column in missing_columns:
logger.info(f"Adding missing column: {column}")
try:
conn.execute(text(f"ALTER TABLE scheduled_tasks ADD COLUMN {column}"))
conn.commit()
logger.info(f"Successfully added column: {column}")
except Exception as alter_error:
logger.error(f"Failed to add column {column}: {alter_error}")
raise
if missing_columns:
logger.info(f"Database schema upgraded with columns: {missing_columns}")
else:
logger.info("Database schema is up to date")
except Exception as e:
logger.error(f"Failed to upgrade database schema: {e}")
logger.error(f"Error type: {type(e)}")
import traceback
logger.error(f"Traceback: {traceback.format_exc()}")
# Don't raise here - let the app continue with create_all
def get_db():
"""
Dependency for FastAPI to get database session.
+8 -3
View File
@@ -83,15 +83,20 @@ async def startup_event():
"""Initialize services on application startup"""
try:
logger.info("Starting up BDFR Web Interface...")
# Initialize database
init_database()
logger.info("Database initialized")
# Ensure database schema is up to date before starting scheduler
from .database import upgrade_database_schema
upgrade_database_schema()
logger.info("Database schema check complete")
# Start scheduler
start_scheduler()
logger.info("Scheduler started")
logger.info("Startup complete!")
except Exception as e:
logger.error(f"Startup error: {e}", exc_info=True)
+8 -2
View File
@@ -56,7 +56,11 @@ class ScheduledTask(Base):
# Authentication
auth_state = Column(String(255), nullable=True)
# User-specific download options
upvoted = Column(Boolean, default=False, nullable=False)
saved = Column(Boolean, default=False, nullable=False)
# Relationships
executions = relationship("TaskExecutionHistory", back_populates="task", cascade="all, delete-orphan")
@@ -82,7 +86,9 @@ class ScheduledTask(Base):
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
"last_run_at": self.last_run_at.isoformat() if self.last_run_at else None,
"next_run_at": self.next_run_at.isoformat() if self.next_run_at else None,
"auth_state": self.auth_state
"auth_state": self.auth_state,
"upvoted": getattr(self, 'upvoted', False),
"saved": getattr(self, 'saved', False)
}
+209 -5
View File
@@ -14,6 +14,7 @@ from .database import get_db
from .models import ScheduledTask, TaskExecutionHistory
from .scheduler import schedule_task, unschedule_task, calculate_next_run
from .task_queue import task_queue
from .auth import get_oauth_manager
logger = logging.getLogger(__name__)
@@ -34,6 +35,8 @@ class ScheduledTaskCreate(BaseModel):
run_time: str = Field(..., pattern="^([01]?[0-9]|2[0-3]):[0-5][0-9]$") # HH:MM format
timezone: str = Field(default="UTC")
auth_state: Optional[str] = None
upvoted: bool = False
saved: bool = False
class ScheduledTaskUpdate(BaseModel):
@@ -67,7 +70,10 @@ class ScheduledTaskResponse(BaseModel):
updated_at: str
last_run_at: Optional[str]
next_run_at: Optional[str]
auth_state: Optional[str]
upvoted: bool
saved: bool
class Config:
from_attributes = True
@@ -124,7 +130,9 @@ async def create_scheduled_task(task_data: ScheduledTaskCreate, db: Session = De
schedule_frequency="daily",
run_time=run_time_obj,
timezone=task_data.timezone,
auth_state=task_data.auth_state
auth_state=task_data.auth_state,
upvoted=task_data.upvoted,
saved=task_data.saved
)
# Calculate next run time
@@ -162,6 +170,10 @@ async def list_scheduled_tasks(db: Session = Depends(get_db)):
tasks = db.query(ScheduledTask).order_by(ScheduledTask.created_at.desc()).all()
return [task.to_dict() for task in tasks]
except Exception as e:
if "no such column" in str(e):
logger.warning(f"Database schema is outdated: {e}")
# Return empty list if schema is outdated
return []
logger.error(f"Failed to list scheduled tasks: {e}", exc_info=True)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
@@ -444,7 +456,7 @@ async def get_queue_status():
"""Get current task queue status"""
try:
status = task_queue.get_queue_status()
# Get details of current task if any
current_task_info = None
if status['current_task']:
@@ -459,16 +471,208 @@ async def get_queue_status():
}
finally:
db.close()
return {
'queue_size': status['queue_size'],
'is_processing': status['is_processing'],
'current_task': current_task_info
}
except Exception as e:
logger.error(f"Failed to get queue status: {e}", exc_info=True)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to get queue status"
)
@router.post("/create-likes", response_model=ScheduledTaskResponse, status_code=status.HTTP_201_CREATED)
async def create_likes_task(
limit: int = 25,
sort: str = "hot",
download_mode: str = "download",
run_now: bool = False,
run_time: str = "02:00",
timezone: str = "UTC",
auth_state: str = None,
db: Session = Depends(get_db)
):
"""
Create a scheduled task to download the user's liked posts.
"""
try:
# Get current username from auth
if not auth_state:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Authentication required"
)
oauth_manager = get_oauth_manager()
auth_status = oauth_manager.get_auth_status(auth_state)
if not auth_status["authenticated"]:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid authentication"
)
username = auth_status["username"]
task_name = f"{username} - Liked posts"
# Validate timezone
try:
pytz.timezone(timezone)
except pytz.exceptions.UnknownTimeZoneError:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Invalid timezone: {timezone}"
)
# Parse run_time
hour, minute = map(int, run_time.split(':'))
run_time_obj = time_type(hour=hour, minute=minute)
# Create task
task = ScheduledTask(
name=task_name,
enabled=True,
source_type="user",
source_name=username,
download_mode=download_mode,
limit=limit,
sort=sort,
time_filter="day",
no_dupes=True,
simple_check=False,
schedule_frequency="daily",
run_time=run_time_obj,
timezone=timezone,
auth_state=auth_state,
upvoted=True,
saved=False
)
# Calculate next run time
task.next_run_at = calculate_next_run(task)
# Save to database
db.add(task)
db.commit()
db.refresh(task)
# Schedule the task
schedule_task(task)
# If run_now is True, queue it immediately
if run_now:
await task_queue.add_task(task.id, priority=1)
logger.info(f"Created likes task {task.id}: {task.name}")
return task.to_dict()
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to create likes task: {e}", exc_info=True)
db.rollback()
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to create likes task: {str(e)}"
)
@router.post("/create-saved", response_model=ScheduledTaskResponse, status_code=status.HTTP_201_CREATED)
async def create_saved_task(
limit: int = 25,
sort: str = "hot",
download_mode: str = "download",
run_now: bool = False,
run_time: str = "02:00",
timezone: str = "UTC",
auth_state: str = None,
db: Session = Depends(get_db)
):
"""
Create a scheduled task to download the user's saved posts.
"""
try:
# Get current username from auth
if not auth_state:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Authentication required"
)
oauth_manager = get_oauth_manager()
auth_status = oauth_manager.get_auth_status(auth_state)
if not auth_status["authenticated"]:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid authentication"
)
username = auth_status["username"]
task_name = f"{username} - Saved posts"
# Validate timezone
try:
pytz.timezone(timezone)
except pytz.exceptions.UnknownTimeZoneError:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Invalid timezone: {timezone}"
)
# Parse run_time
hour, minute = map(int, run_time.split(':'))
run_time_obj = time_type(hour=hour, minute=minute)
# Create task
task = ScheduledTask(
name=task_name,
enabled=True,
source_type="user",
source_name=username,
download_mode=download_mode,
limit=limit,
sort=sort,
time_filter="day",
no_dupes=True,
simple_check=False,
schedule_frequency="daily",
run_time=run_time_obj,
timezone=timezone,
auth_state=auth_state,
upvoted=False,
saved=True
)
# Calculate next run time
task.next_run_at = calculate_next_run(task)
# Save to database
db.add(task)
db.commit()
db.refresh(task)
# Schedule the task
schedule_task(task)
# If run_now is True, queue it immediately
if run_now:
await task_queue.add_task(task.id, priority=1)
logger.info(f"Created saved task {task.id}: {task.name}")
return task.to_dict()
except HTTPException:
raise
except Exception as e:
logger.error(f"Failed to create saved task: {e}", exc_info=True)
db.rollback()
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to create saved task: {str(e)}"
)
+37 -4
View File
@@ -132,12 +132,38 @@ def load_scheduled_tasks():
"""
db = SessionLocal()
try:
tasks = db.query(ScheduledTask).filter(ScheduledTask.enabled == True).all()
# Check if the required columns exist before querying
try:
# First, check if we can access the table at all
db.query(ScheduledTask).first()
# Try to query with the new columns - this will fail if columns don't exist
tasks = db.query(ScheduledTask).filter(ScheduledTask.enabled == True).all()
except Exception as column_error:
if "no such column" in str(column_error):
logger.warning(f"Database schema is outdated. Required columns missing: {column_error}")
logger.info("Attempting to upgrade database schema...")
# Try to upgrade the database
try:
from .database import upgrade_database_schema
upgrade_database_schema()
# Now try again to load tasks
tasks = db.query(ScheduledTask).filter(ScheduledTask.enabled == True).all()
logger.info("Database upgraded successfully, continuing with task loading")
except Exception as upgrade_error:
logger.error(f"Failed to upgrade database: {upgrade_error}")
logger.info("Skipping scheduled task loading until database is manually upgraded")
return
else:
raise
logger.info(f"Loading {len(tasks)} enabled scheduled tasks")
for task in tasks:
schedule_task(task)
logger.info(f"Loaded {len(tasks)} scheduled tasks")
except Exception as e:
logger.error(f"Failed to load scheduled tasks: {e}", exc_info=True)
@@ -198,8 +224,15 @@ async def execute_scheduled_task(task_id: str):
'time_filter': 'day', # Always "day" for daily scheduled tasks
'no_dupes': True, # Always enabled for scheduled tasks
'simple_check': task.simple_check,
'auth_state': task.auth_state
'auth_state': task.auth_state,
'upvoted': task.upvoted,
'saved': task.saved
}
# For user downloads, set submitted appropriately
if task.source_type == 'user':
# For likes or saved, don't download submitted posts
kwargs['submitted'] = not (task.upvoted or task.saved)
# Create download using existing API
download_id = await create_download_with_bdfr_api(
+93
View File
@@ -57,6 +57,13 @@ class BDFRApp {
this.authStateInput = document.getElementById('authState');
this.userAuthStateInput = document.getElementById('userAuthState');
// User downloads section
this.userDownloadsSection = document.getElementById('userDownloadsSection');
this.downloadLikesBtn = document.getElementById('downloadLikesBtn');
this.downloadSavedBtn = document.getElementById('downloadSavedBtn');
this.userScheduleOptions = document.getElementById('userScheduleOptions');
this.userRunTimeInput = document.getElementById('userRunTime');
// Scheduled task form elements
this.runDailyCheckbox = document.getElementById('runDaily');
this.scheduleOptions = document.getElementById('scheduleOptions');
@@ -105,6 +112,20 @@ class BDFRApp {
if (this.logoutBtn) {
this.logoutBtn.addEventListener('click', () => this.handleLogout());
}
// User downloads events
if (this.downloadLikesBtn) {
this.downloadLikesBtn.addEventListener('click', () => this.handleDownloadLikes());
}
if (this.downloadSavedBtn) {
this.downloadSavedBtn.addEventListener('click', () => this.handleDownloadSaved());
}
// User schedule toggle
const userScheduleRadios = document.querySelectorAll('input[name="user_schedule_type"]');
userScheduleRadios.forEach(radio => {
radio.addEventListener('change', (e) => this.updateUserScheduleUI(e.target.value));
});
// Real-time input validation
['subreddit', 'username', 'sourceName'].forEach(id => {
const input = document.getElementById(id);
@@ -886,6 +907,11 @@ class BDFRApp {
if (this.authStateInput) this.authStateInput.value = this.authState || '';
if (this.userAuthStateInput) this.userAuthStateInput.value = this.authState || '';
// Show user downloads section
if (this.userDownloadsSection) {
this.userDownloadsSection.style.display = 'block';
}
} else {
this.authSection.style.display = 'block';
this.authStatus.textContent = '🔴 Not Connected';
@@ -899,6 +925,11 @@ class BDFRApp {
// Clear auth state from forms
if (this.authStateInput) this.authStateInput.value = '';
if (this.userAuthStateInput) this.userAuthStateInput.value = '';
// Hide user downloads section
if (this.userDownloadsSection) {
this.userDownloadsSection.style.display = 'none';
}
}
}
@@ -975,6 +1006,68 @@ class BDFRApp {
}
}
updateUserScheduleUI(scheduleType) {
if (this.userScheduleOptions) {
this.userScheduleOptions.style.display = scheduleType === 'scheduled' ? 'block' : 'none';
}
}
async handleDownloadLikes() {
await this.handleUserDownload('likes');
}
async handleDownloadSaved() {
await this.handleUserDownload('saved');
}
async handleUserDownload(type) {
const downloadMode = document.querySelector('input[name="user_download_mode"]:checked').value;
const scheduleType = document.querySelector('input[name="user_schedule_type"]:checked').value;
const runNow = scheduleType === 'now';
const runTime = this.userRunTimeInput ? this.userRunTimeInput.value : '02:00';
const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
const params = {
limit: 25,
sort: 'hot',
download_mode: downloadMode,
run_now: runNow,
run_time: runTime,
timezone: timezone,
auth_state: this.authState
};
const endpoint = type === 'likes' ? '/api/scheduled-tasks/create-likes' : '/api/scheduled-tasks/create-saved';
try {
this.showLoading(this[`download${type.charAt(0).toUpperCase() + type.slice(1)}Btn`]);
const response = await fetch(endpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(params)
});
const result = await response.json();
if (response.ok) {
const action = runNow ? 'started' : 'scheduled';
this.showSuccess(`Your ${type} download has been ${action}!`);
await this.loadScheduledTasks();
} else {
this.showError(result.detail || `Failed to ${runNow ? 'start' : 'schedule'} ${type} download`);
}
} catch (error) {
console.error('Error:', error);
this.showError(`Network error occurred while ${runNow ? 'starting' : 'scheduling'} ${type} download`);
} finally {
this.hideLoading(this[`download${type.charAt(0).toUpperCase() + type.slice(1)}Btn`]);
}
}
async completeOAuth2Flow(code, state) {
try {
const formData = new FormData();
+76
View File
@@ -28,6 +28,82 @@
</div>
</header>
<!-- User Downloads Section -->
<section class="user-downloads-section" id="userDownloadsSection" style="display: none;">
<div class="form-container-unified">
<div class="form-card-unified">
<h2>📥 My Downloads</h2>
<p>Download your liked and saved posts from Reddit.</p>
<!-- Download Mode Selection -->
<div class="form-section">
<h4>🎯 Download Mode</h4>
<div class="radio-group mode-radio-group">
<label class="radio-label mode-option" data-tooltip="Download media files (images, videos, gifs) from posts">
<input type="radio" name="user_download_mode" value="download" checked>
<span class="radio-custom"></span>
<span class="mode-label">
<strong>Download</strong>
<small>Media files only</small>
</span>
</label>
<label class="radio-label mode-option" data-tooltip="Save post metadata (title, author, comments) as JSON/XML without downloading media">
<input type="radio" name="user_download_mode" value="archive">
<span class="radio-custom"></span>
<span class="mode-label">
<strong>Archive</strong>
<small>Metadata only</small>
</span>
</label>
<label class="radio-label mode-option" data-tooltip="Download media files AND save metadata - complete backup of posts">
<input type="radio" name="user_download_mode" value="clone">
<span class="radio-custom"></span>
<span class="mode-label">
<strong>Clone</strong>
<small>Media + Metadata</small>
</span>
</label>
</div>
</div>
<!-- Scheduling Options -->
<div class="form-section">
<h4>⏰ Scheduling</h4>
<div class="radio-group">
<label class="radio-label">
<input type="radio" name="user_schedule_type" value="now" checked>
<span class="radio-custom"></span>
Run Now
</label>
<label class="radio-label">
<input type="radio" name="user_schedule_type" value="scheduled">
<span class="radio-custom"></span>
Schedule for Later
</label>
</div>
<!-- Scheduled Options (shown when Schedule for Later is selected) -->
<div id="userScheduleOptions" class="schedule-options" style="display: none;">
<div class="form-group">
<label for="userRunTime">Run Time (24-hour format):</label>
<input type="time" id="userRunTime" name="user_run_time" value="02:00">
<small class="form-help">Time to run the download daily (in your local timezone)</small>
</div>
</div>
</div>
<!-- Action Buttons -->
<div class="form-section">
<h4>🚀 Actions</h4>
<div class="user-actions">
<button id="downloadLikesBtn" class="btn btn-primary">❤️ Download My Likes</button>
<button id="downloadSavedBtn" class="btn btn-primary">⭐ Download My Saved Posts</button>
</div>
</div>
</div>
</div>
</section>
<main>
<!-- Unified Download Form Section -->
<section class="download-section">