From 8f8e2c744dbbbc45e849d095ac31e687f356363c Mon Sep 17 00:00:00 2001 From: ModerateWinGuy Date: Thu, 6 Nov 2025 21:53:32 +1300 Subject: [PATCH] fixed file naming thingy --- bdfr/resource.py | 15 +- tests/test_extension_case_normalization.py | 7 + web_interface/app/database.py | 89 +++++++++ web_interface/app/main.py | 11 +- web_interface/app/models.py | 10 +- web_interface/app/scheduled_tasks.py | 214 ++++++++++++++++++++- web_interface/app/scheduler.py | 41 +++- web_interface/static/js/app.js | 93 +++++++++ web_interface/templates/index.html | 76 ++++++++ 9 files changed, 541 insertions(+), 15 deletions(-) diff --git a/bdfr/resource.py b/bdfr/resource.py index 86bdcbe..2eec13f 100644 --- a/bdfr/resource.py +++ b/bdfr/resource.py @@ -24,10 +24,16 @@ class Resource: self.content: Optional[bytes] = None self.url = url self.hash: Optional[_hashlib.HASH] = None + + # Log the original extension before normalization + if extension: + logger.debug(f"Resource constructor received extension: '{extension}' for URL: {url}") + self.extension = self._normalize_extension(extension) self.download_function = download_function if not self.extension: self.extension = self._determine_extension() + logger.debug(f"Extension determined from URL: '{self.extension}' for URL: {url}") @staticmethod def retry_download(url: str) -> Callable: @@ -121,8 +127,15 @@ class Resource: """Normalize extension to lowercase for consistency""" if not extension: return None + + original = extension + # Ensure extension starts with a dot + if not extension.startswith('.'): + extension = '.' + extension + normalized = extension.lower() - logger.debug(f"Normalized extension '{extension}' to '{normalized}'") + if original != normalized: + logger.info(f"Extension normalization: '{original}' -> '{normalized}' for URL: {self.url if hasattr(self, 'url') else 'unknown'}") return normalized @staticmethod diff --git a/tests/test_extension_case_normalization.py b/tests/test_extension_case_normalization.py index 1f5bfe8..01b25ac 100644 --- a/tests/test_extension_case_normalization.py +++ b/tests/test_extension_case_normalization.py @@ -52,6 +52,13 @@ class TestExtensionNormalization: (".JPEG", ".jpeg"), (".PNG", ".png"), (".GIF", ".gif"), + # Test extensions without dots (common from yt-dlp) + ("JPG", ".jpg"), + ("JPEG", ".jpeg"), + ("MP4", ".mp4"), + ("WEBM", ".webm"), + ("mp4", ".mp4"), + ("gif", ".gif"), ] for input_ext, expected in test_cases: diff --git a/web_interface/app/database.py b/web_interface/app/database.py index b9bf7ac..bcc01b5 100644 --- a/web_interface/app/database.py +++ b/web_interface/app/database.py @@ -47,13 +47,102 @@ Base = declarative_base() def init_database(): """Initialize database - called on container startup""" try: + # Check if database file exists + if DB_PATH.exists(): + logger.info(f"Database file exists at {DB_PATH}, checking for schema upgrades") + upgrade_database_schema() + else: + logger.info(f"Creating new database at {DB_PATH}") + Base.metadata.create_all(bind=engine) logger.info(f"Database initialized at {DB_PATH}") + + # Verify the schema after creation/upgrade + verify_database_schema() + except Exception as e: logger.error(f"Failed to initialize database: {e}") raise +def verify_database_schema(): + """Verify that the database schema has all required columns""" + try: + from sqlalchemy import text + + with engine.connect() as conn: + result = conn.execute(text("PRAGMA table_info(scheduled_tasks)")) + columns = [row[1] for row in result.fetchall()] + + required_columns = ['upvoted', 'saved'] + missing_columns = [col for col in required_columns if col not in columns] + + if missing_columns: + logger.error(f"Database schema verification failed. Missing columns: {missing_columns}") + logger.error(f"Available columns: {columns}") + else: + logger.info("Database schema verification passed") + + except Exception as e: + logger.error(f"Failed to verify database schema: {e}") + + +def upgrade_database_schema(): + """Check and upgrade database schema for new columns""" + try: + from sqlalchemy import text + + logger.info("Starting database schema upgrade check...") + + # Check if scheduled_tasks table exists first + with engine.connect() as conn: + # Check if table exists + result = conn.execute(text("SELECT name FROM sqlite_master WHERE type='table' AND name='scheduled_tasks'")) + table_exists = result.fetchone() is not None + + if not table_exists: + logger.info("scheduled_tasks table does not exist, will be created by create_all") + return + + logger.info("scheduled_tasks table exists, checking columns...") + + # Get table info + result = conn.execute(text("PRAGMA table_info(scheduled_tasks)")) + columns = [row[1] for row in result.fetchall()] + + logger.info(f"Existing columns in scheduled_tasks: {columns}") + + # Check for missing columns + missing_columns = [] + if 'upvoted' not in columns: + missing_columns.append('upvoted BOOLEAN DEFAULT 0') + if 'saved' not in columns: + missing_columns.append('saved BOOLEAN DEFAULT 0') + + # Add missing columns + for column in missing_columns: + logger.info(f"Adding missing column: {column}") + try: + conn.execute(text(f"ALTER TABLE scheduled_tasks ADD COLUMN {column}")) + conn.commit() + logger.info(f"Successfully added column: {column}") + except Exception as alter_error: + logger.error(f"Failed to add column {column}: {alter_error}") + raise + + if missing_columns: + logger.info(f"Database schema upgraded with columns: {missing_columns}") + else: + logger.info("Database schema is up to date") + + except Exception as e: + logger.error(f"Failed to upgrade database schema: {e}") + logger.error(f"Error type: {type(e)}") + import traceback + logger.error(f"Traceback: {traceback.format_exc()}") + # Don't raise here - let the app continue with create_all + + def get_db(): """ Dependency for FastAPI to get database session. diff --git a/web_interface/app/main.py b/web_interface/app/main.py index 9f4c9ec..9741281 100644 --- a/web_interface/app/main.py +++ b/web_interface/app/main.py @@ -83,15 +83,20 @@ async def startup_event(): """Initialize services on application startup""" try: logger.info("Starting up BDFR Web Interface...") - + # Initialize database init_database() logger.info("Database initialized") - + + # Ensure database schema is up to date before starting scheduler + from .database import upgrade_database_schema + upgrade_database_schema() + logger.info("Database schema check complete") + # Start scheduler start_scheduler() logger.info("Scheduler started") - + logger.info("Startup complete!") except Exception as e: logger.error(f"Startup error: {e}", exc_info=True) diff --git a/web_interface/app/models.py b/web_interface/app/models.py index 147e2a6..1f330b3 100644 --- a/web_interface/app/models.py +++ b/web_interface/app/models.py @@ -56,7 +56,11 @@ class ScheduledTask(Base): # Authentication auth_state = Column(String(255), nullable=True) - + + # User-specific download options + upvoted = Column(Boolean, default=False, nullable=False) + saved = Column(Boolean, default=False, nullable=False) + # Relationships executions = relationship("TaskExecutionHistory", back_populates="task", cascade="all, delete-orphan") @@ -82,7 +86,9 @@ class ScheduledTask(Base): "updated_at": self.updated_at.isoformat() if self.updated_at else None, "last_run_at": self.last_run_at.isoformat() if self.last_run_at else None, "next_run_at": self.next_run_at.isoformat() if self.next_run_at else None, - "auth_state": self.auth_state + "auth_state": self.auth_state, + "upvoted": getattr(self, 'upvoted', False), + "saved": getattr(self, 'saved', False) } diff --git a/web_interface/app/scheduled_tasks.py b/web_interface/app/scheduled_tasks.py index 29197bf..d1265ea 100644 --- a/web_interface/app/scheduled_tasks.py +++ b/web_interface/app/scheduled_tasks.py @@ -14,6 +14,7 @@ from .database import get_db from .models import ScheduledTask, TaskExecutionHistory from .scheduler import schedule_task, unschedule_task, calculate_next_run from .task_queue import task_queue +from .auth import get_oauth_manager logger = logging.getLogger(__name__) @@ -34,6 +35,8 @@ class ScheduledTaskCreate(BaseModel): run_time: str = Field(..., pattern="^([01]?[0-9]|2[0-3]):[0-5][0-9]$") # HH:MM format timezone: str = Field(default="UTC") auth_state: Optional[str] = None + upvoted: bool = False + saved: bool = False class ScheduledTaskUpdate(BaseModel): @@ -67,7 +70,10 @@ class ScheduledTaskResponse(BaseModel): updated_at: str last_run_at: Optional[str] next_run_at: Optional[str] - + auth_state: Optional[str] + upvoted: bool + saved: bool + class Config: from_attributes = True @@ -124,7 +130,9 @@ async def create_scheduled_task(task_data: ScheduledTaskCreate, db: Session = De schedule_frequency="daily", run_time=run_time_obj, timezone=task_data.timezone, - auth_state=task_data.auth_state + auth_state=task_data.auth_state, + upvoted=task_data.upvoted, + saved=task_data.saved ) # Calculate next run time @@ -162,6 +170,10 @@ async def list_scheduled_tasks(db: Session = Depends(get_db)): tasks = db.query(ScheduledTask).order_by(ScheduledTask.created_at.desc()).all() return [task.to_dict() for task in tasks] except Exception as e: + if "no such column" in str(e): + logger.warning(f"Database schema is outdated: {e}") + # Return empty list if schema is outdated + return [] logger.error(f"Failed to list scheduled tasks: {e}", exc_info=True) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -444,7 +456,7 @@ async def get_queue_status(): """Get current task queue status""" try: status = task_queue.get_queue_status() - + # Get details of current task if any current_task_info = None if status['current_task']: @@ -459,16 +471,208 @@ async def get_queue_status(): } finally: db.close() - + return { 'queue_size': status['queue_size'], 'is_processing': status['is_processing'], 'current_task': current_task_info } - + except Exception as e: logger.error(f"Failed to get queue status: {e}", exc_info=True) raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to get queue status" + ) + + +@router.post("/create-likes", response_model=ScheduledTaskResponse, status_code=status.HTTP_201_CREATED) +async def create_likes_task( + limit: int = 25, + sort: str = "hot", + download_mode: str = "download", + run_now: bool = False, + run_time: str = "02:00", + timezone: str = "UTC", + auth_state: str = None, + db: Session = Depends(get_db) +): + """ + Create a scheduled task to download the user's liked posts. + """ + try: + # Get current username from auth + if not auth_state: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Authentication required" + ) + + oauth_manager = get_oauth_manager() + auth_status = oauth_manager.get_auth_status(auth_state) + if not auth_status["authenticated"]: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid authentication" + ) + + username = auth_status["username"] + task_name = f"{username} - Liked posts" + + # Validate timezone + try: + pytz.timezone(timezone) + except pytz.exceptions.UnknownTimeZoneError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid timezone: {timezone}" + ) + + # Parse run_time + hour, minute = map(int, run_time.split(':')) + run_time_obj = time_type(hour=hour, minute=minute) + + # Create task + task = ScheduledTask( + name=task_name, + enabled=True, + source_type="user", + source_name=username, + download_mode=download_mode, + limit=limit, + sort=sort, + time_filter="day", + no_dupes=True, + simple_check=False, + schedule_frequency="daily", + run_time=run_time_obj, + timezone=timezone, + auth_state=auth_state, + upvoted=True, + saved=False + ) + + # Calculate next run time + task.next_run_at = calculate_next_run(task) + + # Save to database + db.add(task) + db.commit() + db.refresh(task) + + # Schedule the task + schedule_task(task) + + # If run_now is True, queue it immediately + if run_now: + await task_queue.add_task(task.id, priority=1) + + logger.info(f"Created likes task {task.id}: {task.name}") + + return task.to_dict() + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to create likes task: {e}", exc_info=True) + db.rollback() + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to create likes task: {str(e)}" + ) + + +@router.post("/create-saved", response_model=ScheduledTaskResponse, status_code=status.HTTP_201_CREATED) +async def create_saved_task( + limit: int = 25, + sort: str = "hot", + download_mode: str = "download", + run_now: bool = False, + run_time: str = "02:00", + timezone: str = "UTC", + auth_state: str = None, + db: Session = Depends(get_db) +): + """ + Create a scheduled task to download the user's saved posts. + """ + try: + # Get current username from auth + if not auth_state: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Authentication required" + ) + + oauth_manager = get_oauth_manager() + auth_status = oauth_manager.get_auth_status(auth_state) + if not auth_status["authenticated"]: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid authentication" + ) + + username = auth_status["username"] + task_name = f"{username} - Saved posts" + + # Validate timezone + try: + pytz.timezone(timezone) + except pytz.exceptions.UnknownTimeZoneError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid timezone: {timezone}" + ) + + # Parse run_time + hour, minute = map(int, run_time.split(':')) + run_time_obj = time_type(hour=hour, minute=minute) + + # Create task + task = ScheduledTask( + name=task_name, + enabled=True, + source_type="user", + source_name=username, + download_mode=download_mode, + limit=limit, + sort=sort, + time_filter="day", + no_dupes=True, + simple_check=False, + schedule_frequency="daily", + run_time=run_time_obj, + timezone=timezone, + auth_state=auth_state, + upvoted=False, + saved=True + ) + + # Calculate next run time + task.next_run_at = calculate_next_run(task) + + # Save to database + db.add(task) + db.commit() + db.refresh(task) + + # Schedule the task + schedule_task(task) + + # If run_now is True, queue it immediately + if run_now: + await task_queue.add_task(task.id, priority=1) + + logger.info(f"Created saved task {task.id}: {task.name}") + + return task.to_dict() + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to create saved task: {e}", exc_info=True) + db.rollback() + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to create saved task: {str(e)}" ) \ No newline at end of file diff --git a/web_interface/app/scheduler.py b/web_interface/app/scheduler.py index 1d0cca3..cfe6cf3 100644 --- a/web_interface/app/scheduler.py +++ b/web_interface/app/scheduler.py @@ -132,12 +132,38 @@ def load_scheduled_tasks(): """ db = SessionLocal() try: - tasks = db.query(ScheduledTask).filter(ScheduledTask.enabled == True).all() + # Check if the required columns exist before querying + try: + # First, check if we can access the table at all + db.query(ScheduledTask).first() + + # Try to query with the new columns - this will fail if columns don't exist + tasks = db.query(ScheduledTask).filter(ScheduledTask.enabled == True).all() + except Exception as column_error: + if "no such column" in str(column_error): + logger.warning(f"Database schema is outdated. Required columns missing: {column_error}") + logger.info("Attempting to upgrade database schema...") + + # Try to upgrade the database + try: + from .database import upgrade_database_schema + upgrade_database_schema() + + # Now try again to load tasks + tasks = db.query(ScheduledTask).filter(ScheduledTask.enabled == True).all() + logger.info("Database upgraded successfully, continuing with task loading") + except Exception as upgrade_error: + logger.error(f"Failed to upgrade database: {upgrade_error}") + logger.info("Skipping scheduled task loading until database is manually upgraded") + return + else: + raise + logger.info(f"Loading {len(tasks)} enabled scheduled tasks") - + for task in tasks: schedule_task(task) - + logger.info(f"Loaded {len(tasks)} scheduled tasks") except Exception as e: logger.error(f"Failed to load scheduled tasks: {e}", exc_info=True) @@ -198,8 +224,15 @@ async def execute_scheduled_task(task_id: str): 'time_filter': 'day', # Always "day" for daily scheduled tasks 'no_dupes': True, # Always enabled for scheduled tasks 'simple_check': task.simple_check, - 'auth_state': task.auth_state + 'auth_state': task.auth_state, + 'upvoted': task.upvoted, + 'saved': task.saved } + + # For user downloads, set submitted appropriately + if task.source_type == 'user': + # For likes or saved, don't download submitted posts + kwargs['submitted'] = not (task.upvoted or task.saved) # Create download using existing API download_id = await create_download_with_bdfr_api( diff --git a/web_interface/static/js/app.js b/web_interface/static/js/app.js index dc87426..d296e00 100644 --- a/web_interface/static/js/app.js +++ b/web_interface/static/js/app.js @@ -57,6 +57,13 @@ class BDFRApp { this.authStateInput = document.getElementById('authState'); this.userAuthStateInput = document.getElementById('userAuthState'); + // User downloads section + this.userDownloadsSection = document.getElementById('userDownloadsSection'); + this.downloadLikesBtn = document.getElementById('downloadLikesBtn'); + this.downloadSavedBtn = document.getElementById('downloadSavedBtn'); + this.userScheduleOptions = document.getElementById('userScheduleOptions'); + this.userRunTimeInput = document.getElementById('userRunTime'); + // Scheduled task form elements this.runDailyCheckbox = document.getElementById('runDaily'); this.scheduleOptions = document.getElementById('scheduleOptions'); @@ -105,6 +112,20 @@ class BDFRApp { if (this.logoutBtn) { this.logoutBtn.addEventListener('click', () => this.handleLogout()); } + + // User downloads events + if (this.downloadLikesBtn) { + this.downloadLikesBtn.addEventListener('click', () => this.handleDownloadLikes()); + } + if (this.downloadSavedBtn) { + this.downloadSavedBtn.addEventListener('click', () => this.handleDownloadSaved()); + } + + // User schedule toggle + const userScheduleRadios = document.querySelectorAll('input[name="user_schedule_type"]'); + userScheduleRadios.forEach(radio => { + radio.addEventListener('change', (e) => this.updateUserScheduleUI(e.target.value)); + }); // Real-time input validation ['subreddit', 'username', 'sourceName'].forEach(id => { const input = document.getElementById(id); @@ -886,6 +907,11 @@ class BDFRApp { if (this.authStateInput) this.authStateInput.value = this.authState || ''; if (this.userAuthStateInput) this.userAuthStateInput.value = this.authState || ''; + // Show user downloads section + if (this.userDownloadsSection) { + this.userDownloadsSection.style.display = 'block'; + } + } else { this.authSection.style.display = 'block'; this.authStatus.textContent = '🔴 Not Connected'; @@ -899,6 +925,11 @@ class BDFRApp { // Clear auth state from forms if (this.authStateInput) this.authStateInput.value = ''; if (this.userAuthStateInput) this.userAuthStateInput.value = ''; + + // Hide user downloads section + if (this.userDownloadsSection) { + this.userDownloadsSection.style.display = 'none'; + } } } @@ -975,6 +1006,68 @@ class BDFRApp { } } + updateUserScheduleUI(scheduleType) { + if (this.userScheduleOptions) { + this.userScheduleOptions.style.display = scheduleType === 'scheduled' ? 'block' : 'none'; + } + } + + async handleDownloadLikes() { + await this.handleUserDownload('likes'); + } + + async handleDownloadSaved() { + await this.handleUserDownload('saved'); + } + + async handleUserDownload(type) { + const downloadMode = document.querySelector('input[name="user_download_mode"]:checked').value; + const scheduleType = document.querySelector('input[name="user_schedule_type"]:checked').value; + const runNow = scheduleType === 'now'; + const runTime = this.userRunTimeInput ? this.userRunTimeInput.value : '02:00'; + const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone; + + const params = { + limit: 25, + sort: 'hot', + download_mode: downloadMode, + run_now: runNow, + run_time: runTime, + timezone: timezone, + auth_state: this.authState + }; + + const endpoint = type === 'likes' ? '/api/scheduled-tasks/create-likes' : '/api/scheduled-tasks/create-saved'; + + try { + this.showLoading(this[`download${type.charAt(0).toUpperCase() + type.slice(1)}Btn`]); + + const response = await fetch(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(params) + }); + + const result = await response.json(); + + if (response.ok) { + const action = runNow ? 'started' : 'scheduled'; + this.showSuccess(`Your ${type} download has been ${action}!`); + await this.loadScheduledTasks(); + } else { + this.showError(result.detail || `Failed to ${runNow ? 'start' : 'schedule'} ${type} download`); + } + + } catch (error) { + console.error('Error:', error); + this.showError(`Network error occurred while ${runNow ? 'starting' : 'scheduling'} ${type} download`); + } finally { + this.hideLoading(this[`download${type.charAt(0).toUpperCase() + type.slice(1)}Btn`]); + } + } + async completeOAuth2Flow(code, state) { try { const formData = new FormData(); diff --git a/web_interface/templates/index.html b/web_interface/templates/index.html index 28ed443..9c44b05 100644 --- a/web_interface/templates/index.html +++ b/web_interface/templates/index.html @@ -28,6 +28,82 @@ + + +