fixed file naming thingy

bdfr/resource.py

@@ -24,10 +24,16 @@ class Resource:
         self.content: Optional[bytes] = None
         self.url = url
         self.hash: Optional[_hashlib.HASH] = None
+        
+        # Log the original extension before normalization
+        if extension:
+            logger.debug(f"Resource constructor received extension: '{extension}' for URL: {url}")
+        
         self.extension = self._normalize_extension(extension)
         self.download_function = download_function
         if not self.extension:
             self.extension = self._determine_extension()
+            logger.debug(f"Extension determined from URL: '{self.extension}' for URL: {url}")
 
     @staticmethod
     def retry_download(url: str) -> Callable:
@@ -121,8 +127,15 @@ class Resource:
         """Normalize extension to lowercase for consistency"""
         if not extension:
             return None
+        
+        original = extension
+        # Ensure extension starts with a dot
+        if not extension.startswith('.'):
+            extension = '.' + extension
+        
         normalized = extension.lower()
-        logger.debug(f"Normalized extension '{extension}' to '{normalized}'")
+        if original != normalized:
+            logger.info(f"Extension normalization: '{original}' -> '{normalized}' for URL: {self.url if hasattr(self, 'url') else 'unknown'}")
         return normalized
 
     @staticmethod

tests/test_extension_case_normalization.py

@@ -52,6 +52,13 @@ class TestExtensionNormalization:
             (".JPEG", ".jpeg"),
             (".PNG", ".png"),
             (".GIF", ".gif"),
+            # Test extensions without dots (common from yt-dlp)
+            ("JPG", ".jpg"),
+            ("JPEG", ".jpeg"),
+            ("MP4", ".mp4"),
+            ("WEBM", ".webm"),
+            ("mp4", ".mp4"),
+            ("gif", ".gif"),
         ]
 
         for input_ext, expected in test_cases:

web_interface/app/database.py

@@ -47,13 +47,102 @@ Base = declarative_base()
 def init_database():
     """Initialize database - called on container startup"""
     try:
+        # Check if database file exists
+        if DB_PATH.exists():
+            logger.info(f"Database file exists at {DB_PATH}, checking for schema upgrades")
+            upgrade_database_schema()
+        else:
+            logger.info(f"Creating new database at {DB_PATH}")
+
         Base.metadata.create_all(bind=engine)
         logger.info(f"Database initialized at {DB_PATH}")
+
+        # Verify the schema after creation/upgrade
+        verify_database_schema()
+
     except Exception as e:
         logger.error(f"Failed to initialize database: {e}")
         raise
 
 
+def verify_database_schema():
+    """Verify that the database schema has all required columns"""
+    try:
+        from sqlalchemy import text
+
+        with engine.connect() as conn:
+            result = conn.execute(text("PRAGMA table_info(scheduled_tasks)"))
+            columns = [row[1] for row in result.fetchall()]
+
+            required_columns = ['upvoted', 'saved']
+            missing_columns = [col for col in required_columns if col not in columns]
+
+            if missing_columns:
+                logger.error(f"Database schema verification failed. Missing columns: {missing_columns}")
+                logger.error(f"Available columns: {columns}")
+            else:
+                logger.info("Database schema verification passed")
+
+    except Exception as e:
+        logger.error(f"Failed to verify database schema: {e}")
+
+
+def upgrade_database_schema():
+    """Check and upgrade database schema for new columns"""
+    try:
+        from sqlalchemy import text
+
+        logger.info("Starting database schema upgrade check...")
+
+        # Check if scheduled_tasks table exists first
+        with engine.connect() as conn:
+            # Check if table exists
+            result = conn.execute(text("SELECT name FROM sqlite_master WHERE type='table' AND name='scheduled_tasks'"))
+            table_exists = result.fetchone() is not None
+
+            if not table_exists:
+                logger.info("scheduled_tasks table does not exist, will be created by create_all")
+                return
+
+            logger.info("scheduled_tasks table exists, checking columns...")
+
+            # Get table info
+            result = conn.execute(text("PRAGMA table_info(scheduled_tasks)"))
+            columns = [row[1] for row in result.fetchall()]
+
+            logger.info(f"Existing columns in scheduled_tasks: {columns}")
+
+            # Check for missing columns
+            missing_columns = []
+            if 'upvoted' not in columns:
+                missing_columns.append('upvoted BOOLEAN DEFAULT 0')
+            if 'saved' not in columns:
+                missing_columns.append('saved BOOLEAN DEFAULT 0')
+
+            # Add missing columns
+            for column in missing_columns:
+                logger.info(f"Adding missing column: {column}")
+                try:
+                    conn.execute(text(f"ALTER TABLE scheduled_tasks ADD COLUMN {column}"))
+                    conn.commit()
+                    logger.info(f"Successfully added column: {column}")
+                except Exception as alter_error:
+                    logger.error(f"Failed to add column {column}: {alter_error}")
+                    raise
+
+            if missing_columns:
+                logger.info(f"Database schema upgraded with columns: {missing_columns}")
+            else:
+                logger.info("Database schema is up to date")
+
+    except Exception as e:
+        logger.error(f"Failed to upgrade database schema: {e}")
+        logger.error(f"Error type: {type(e)}")
+        import traceback
+        logger.error(f"Traceback: {traceback.format_exc()}")
+        # Don't raise here - let the app continue with create_all
+
+
 def get_db():
     """
     Dependency for FastAPI to get database session.

web_interface/app/main.py

@@ -88,6 +88,11 @@ async def startup_event():
         init_database()
         logger.info("Database initialized")
 
+        # Ensure database schema is up to date before starting scheduler
+        from .database import upgrade_database_schema
+        upgrade_database_schema()
+        logger.info("Database schema check complete")
+
         # Start scheduler
         start_scheduler()
         logger.info("Scheduler started")

web_interface/app/models.py

@@ -57,6 +57,10 @@ class ScheduledTask(Base):
     # Authentication
     auth_state = Column(String(255), nullable=True)
 
+    # User-specific download options
+    upvoted = Column(Boolean, default=False, nullable=False)
+    saved = Column(Boolean, default=False, nullable=False)
+
     # Relationships
     executions = relationship("TaskExecutionHistory", back_populates="task", cascade="all, delete-orphan")
     
@@ -82,7 +86,9 @@ class ScheduledTask(Base):
             "updated_at": self.updated_at.isoformat() if self.updated_at else None,
             "last_run_at": self.last_run_at.isoformat() if self.last_run_at else None,
             "next_run_at": self.next_run_at.isoformat() if self.next_run_at else None,
-            "auth_state": self.auth_state
+            "auth_state": self.auth_state,
+            "upvoted": getattr(self, 'upvoted', False),
+            "saved": getattr(self, 'saved', False)
         }
 
 

web_interface/app/scheduled_tasks.py

@@ -14,6 +14,7 @@ from .database import get_db
 from .models import ScheduledTask, TaskExecutionHistory
 from .scheduler import schedule_task, unschedule_task, calculate_next_run
 from .task_queue import task_queue
+from .auth import get_oauth_manager
 
 logger = logging.getLogger(__name__)
 
@@ -34,6 +35,8 @@ class ScheduledTaskCreate(BaseModel):
     run_time: str = Field(..., pattern="^([01]?[0-9]|2[0-3]):[0-5][0-9]$")  # HH:MM format
     timezone: str = Field(default="UTC")
     auth_state: Optional[str] = None
+    upvoted: bool = False
+    saved: bool = False
 
 
 class ScheduledTaskUpdate(BaseModel):
@@ -67,6 +70,9 @@ class ScheduledTaskResponse(BaseModel):
     updated_at: str
     last_run_at: Optional[str]
     next_run_at: Optional[str]
+    auth_state: Optional[str]
+    upvoted: bool
+    saved: bool
 
     class Config:
         from_attributes = True
@@ -124,7 +130,9 @@ async def create_scheduled_task(task_data: ScheduledTaskCreate, db: Session = De
             schedule_frequency="daily",
             run_time=run_time_obj,
             timezone=task_data.timezone,
-            auth_state=task_data.auth_state
+            auth_state=task_data.auth_state,
+            upvoted=task_data.upvoted,
+            saved=task_data.saved
         )
         
         # Calculate next run time
@@ -162,6 +170,10 @@ async def list_scheduled_tasks(db: Session = Depends(get_db)):
         tasks = db.query(ScheduledTask).order_by(ScheduledTask.created_at.desc()).all()
         return [task.to_dict() for task in tasks]
     except Exception as e:
+        if "no such column" in str(e):
+            logger.warning(f"Database schema is outdated: {e}")
+            # Return empty list if schema is outdated
+            return []
         logger.error(f"Failed to list scheduled tasks: {e}", exc_info=True)
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
@@ -472,3 +484,195 @@ async def get_queue_status():
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
             detail="Failed to get queue status"
         )
+
+
+@router.post("/create-likes", response_model=ScheduledTaskResponse, status_code=status.HTTP_201_CREATED)
+async def create_likes_task(
+    limit: int = 25,
+    sort: str = "hot",
+    download_mode: str = "download",
+    run_now: bool = False,
+    run_time: str = "02:00",
+    timezone: str = "UTC",
+    auth_state: str = None,
+    db: Session = Depends(get_db)
+):
+    """
+    Create a scheduled task to download the user's liked posts.
+    """
+    try:
+        # Get current username from auth
+        if not auth_state:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail="Authentication required"
+            )
+
+        oauth_manager = get_oauth_manager()
+        auth_status = oauth_manager.get_auth_status(auth_state)
+        if not auth_status["authenticated"]:
+            raise HTTPException(
+                status_code=status.HTTP_401_UNAUTHORIZED,
+                detail="Invalid authentication"
+            )
+
+        username = auth_status["username"]
+        task_name = f"{username} - Liked posts"
+
+        # Validate timezone
+        try:
+            pytz.timezone(timezone)
+        except pytz.exceptions.UnknownTimeZoneError:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=f"Invalid timezone: {timezone}"
+            )
+
+        # Parse run_time
+        hour, minute = map(int, run_time.split(':'))
+        run_time_obj = time_type(hour=hour, minute=minute)
+
+        # Create task
+        task = ScheduledTask(
+            name=task_name,
+            enabled=True,
+            source_type="user",
+            source_name=username,
+            download_mode=download_mode,
+            limit=limit,
+            sort=sort,
+            time_filter="day",
+            no_dupes=True,
+            simple_check=False,
+            schedule_frequency="daily",
+            run_time=run_time_obj,
+            timezone=timezone,
+            auth_state=auth_state,
+            upvoted=True,
+            saved=False
+        )
+
+        # Calculate next run time
+        task.next_run_at = calculate_next_run(task)
+
+        # Save to database
+        db.add(task)
+        db.commit()
+        db.refresh(task)
+
+        # Schedule the task
+        schedule_task(task)
+
+        # If run_now is True, queue it immediately
+        if run_now:
+            await task_queue.add_task(task.id, priority=1)
+
+        logger.info(f"Created likes task {task.id}: {task.name}")
+
+        return task.to_dict()
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to create likes task: {e}", exc_info=True)
+        db.rollback()
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to create likes task: {str(e)}"
+        )
+
+
+@router.post("/create-saved", response_model=ScheduledTaskResponse, status_code=status.HTTP_201_CREATED)
+async def create_saved_task(
+    limit: int = 25,
+    sort: str = "hot",
+    download_mode: str = "download",
+    run_now: bool = False,
+    run_time: str = "02:00",
+    timezone: str = "UTC",
+    auth_state: str = None,
+    db: Session = Depends(get_db)
+):
+    """
+    Create a scheduled task to download the user's saved posts.
+    """
+    try:
+        # Get current username from auth
+        if not auth_state:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail="Authentication required"
+            )
+
+        oauth_manager = get_oauth_manager()
+        auth_status = oauth_manager.get_auth_status(auth_state)
+        if not auth_status["authenticated"]:
+            raise HTTPException(
+                status_code=status.HTTP_401_UNAUTHORIZED,
+                detail="Invalid authentication"
+            )
+
+        username = auth_status["username"]
+        task_name = f"{username} - Saved posts"
+
+        # Validate timezone
+        try:
+            pytz.timezone(timezone)
+        except pytz.exceptions.UnknownTimeZoneError:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=f"Invalid timezone: {timezone}"
+            )
+
+        # Parse run_time
+        hour, minute = map(int, run_time.split(':'))
+        run_time_obj = time_type(hour=hour, minute=minute)
+
+        # Create task
+        task = ScheduledTask(
+            name=task_name,
+            enabled=True,
+            source_type="user",
+            source_name=username,
+            download_mode=download_mode,
+            limit=limit,
+            sort=sort,
+            time_filter="day",
+            no_dupes=True,
+            simple_check=False,
+            schedule_frequency="daily",
+            run_time=run_time_obj,
+            timezone=timezone,
+            auth_state=auth_state,
+            upvoted=False,
+            saved=True
+        )
+
+        # Calculate next run time
+        task.next_run_at = calculate_next_run(task)
+
+        # Save to database
+        db.add(task)
+        db.commit()
+        db.refresh(task)
+
+        # Schedule the task
+        schedule_task(task)
+
+        # If run_now is True, queue it immediately
+        if run_now:
+            await task_queue.add_task(task.id, priority=1)
+
+        logger.info(f"Created saved task {task.id}: {task.name}")
+
+        return task.to_dict()
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to create saved task: {e}", exc_info=True)
+        db.rollback()
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to create saved task: {str(e)}"
+        )

web_interface/app/scheduler.py

@@ -132,7 +132,33 @@ def load_scheduled_tasks():
     """
     db = SessionLocal()
     try:
-        tasks = db.query(ScheduledTask).filter(ScheduledTask.enabled == True).all()
+        # Check if the required columns exist before querying
+        try:
+            # First, check if we can access the table at all
+            db.query(ScheduledTask).first()
+
+            # Try to query with the new columns - this will fail if columns don't exist
+            tasks = db.query(ScheduledTask).filter(ScheduledTask.enabled == True).all()
+        except Exception as column_error:
+            if "no such column" in str(column_error):
+                logger.warning(f"Database schema is outdated. Required columns missing: {column_error}")
+                logger.info("Attempting to upgrade database schema...")
+
+                # Try to upgrade the database
+                try:
+                    from .database import upgrade_database_schema
+                    upgrade_database_schema()
+
+                    # Now try again to load tasks
+                    tasks = db.query(ScheduledTask).filter(ScheduledTask.enabled == True).all()
+                    logger.info("Database upgraded successfully, continuing with task loading")
+                except Exception as upgrade_error:
+                    logger.error(f"Failed to upgrade database: {upgrade_error}")
+                    logger.info("Skipping scheduled task loading until database is manually upgraded")
+                    return
+            else:
+                raise
+
         logger.info(f"Loading {len(tasks)} enabled scheduled tasks")
 
         for task in tasks:
@@ -198,9 +224,16 @@ async def execute_scheduled_task(task_id: str):
                 'time_filter': 'day',  # Always "day" for daily scheduled tasks
                 'no_dupes': True,  # Always enabled for scheduled tasks
                 'simple_check': task.simple_check,
-                'auth_state': task.auth_state
+                'auth_state': task.auth_state,
+                'upvoted': task.upvoted,
+                'saved': task.saved
             }
 
+            # For user downloads, set submitted appropriately
+            if task.source_type == 'user':
+                # For likes or saved, don't download submitted posts
+                kwargs['submitted'] = not (task.upvoted or task.saved)
+            
             # Create download using existing API
             download_id = await create_download_with_bdfr_api(
                 download_type=task.source_type,

web_interface/static/js/app.js

@@ -57,6 +57,13 @@ class BDFRApp {
         this.authStateInput = document.getElementById('authState');
         this.userAuthStateInput = document.getElementById('userAuthState');
 
+        // User downloads section
+        this.userDownloadsSection = document.getElementById('userDownloadsSection');
+        this.downloadLikesBtn = document.getElementById('downloadLikesBtn');
+        this.downloadSavedBtn = document.getElementById('downloadSavedBtn');
+        this.userScheduleOptions = document.getElementById('userScheduleOptions');
+        this.userRunTimeInput = document.getElementById('userRunTime');
+
         // Scheduled task form elements
         this.runDailyCheckbox = document.getElementById('runDaily');
         this.scheduleOptions = document.getElementById('scheduleOptions');
@@ -105,6 +112,20 @@ class BDFRApp {
         if (this.logoutBtn) {
             this.logoutBtn.addEventListener('click', () => this.handleLogout());
         }
+
+        // User downloads events
+        if (this.downloadLikesBtn) {
+            this.downloadLikesBtn.addEventListener('click', () => this.handleDownloadLikes());
+        }
+        if (this.downloadSavedBtn) {
+            this.downloadSavedBtn.addEventListener('click', () => this.handleDownloadSaved());
+        }
+
+        // User schedule toggle
+        const userScheduleRadios = document.querySelectorAll('input[name="user_schedule_type"]');
+        userScheduleRadios.forEach(radio => {
+            radio.addEventListener('change', (e) => this.updateUserScheduleUI(e.target.value));
+        });
         // Real-time input validation
         ['subreddit', 'username', 'sourceName'].forEach(id => {
             const input = document.getElementById(id);
@@ -886,6 +907,11 @@ class BDFRApp {
             if (this.authStateInput) this.authStateInput.value = this.authState || '';
             if (this.userAuthStateInput) this.userAuthStateInput.value = this.authState || '';
 
+            // Show user downloads section
+            if (this.userDownloadsSection) {
+                this.userDownloadsSection.style.display = 'block';
+            }
+
         } else {
             this.authSection.style.display = 'block';
             this.authStatus.textContent = '🔴 Not Connected';
@@ -899,6 +925,11 @@ class BDFRApp {
             // Clear auth state from forms
             if (this.authStateInput) this.authStateInput.value = '';
             if (this.userAuthStateInput) this.userAuthStateInput.value = '';
+
+            // Hide user downloads section
+            if (this.userDownloadsSection) {
+                this.userDownloadsSection.style.display = 'none';
+            }
         }
     }
 
@@ -975,6 +1006,68 @@ class BDFRApp {
         }
     }
 
+    updateUserScheduleUI(scheduleType) {
+        if (this.userScheduleOptions) {
+            this.userScheduleOptions.style.display = scheduleType === 'scheduled' ? 'block' : 'none';
+        }
+    }
+
+    async handleDownloadLikes() {
+        await this.handleUserDownload('likes');
+    }
+
+    async handleDownloadSaved() {
+        await this.handleUserDownload('saved');
+    }
+
+    async handleUserDownload(type) {
+        const downloadMode = document.querySelector('input[name="user_download_mode"]:checked').value;
+        const scheduleType = document.querySelector('input[name="user_schedule_type"]:checked').value;
+        const runNow = scheduleType === 'now';
+        const runTime = this.userRunTimeInput ? this.userRunTimeInput.value : '02:00';
+        const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+
+        const params = {
+            limit: 25,
+            sort: 'hot',
+            download_mode: downloadMode,
+            run_now: runNow,
+            run_time: runTime,
+            timezone: timezone,
+            auth_state: this.authState
+        };
+
+        const endpoint = type === 'likes' ? '/api/scheduled-tasks/create-likes' : '/api/scheduled-tasks/create-saved';
+
+        try {
+            this.showLoading(this[`download${type.charAt(0).toUpperCase() + type.slice(1)}Btn`]);
+
+            const response = await fetch(endpoint, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json'
+                },
+                body: JSON.stringify(params)
+            });
+
+            const result = await response.json();
+
+            if (response.ok) {
+                const action = runNow ? 'started' : 'scheduled';
+                this.showSuccess(`Your ${type} download has been ${action}!`);
+                await this.loadScheduledTasks();
+            } else {
+                this.showError(result.detail || `Failed to ${runNow ? 'start' : 'schedule'} ${type} download`);
+            }
+
+        } catch (error) {
+            console.error('Error:', error);
+            this.showError(`Network error occurred while ${runNow ? 'starting' : 'scheduling'} ${type} download`);
+        } finally {
+            this.hideLoading(this[`download${type.charAt(0).toUpperCase() + type.slice(1)}Btn`]);
+        }
+    }
+
     async completeOAuth2Flow(code, state) {
         try {
             const formData = new FormData();

web_interface/templates/index.html

@@ -28,6 +28,82 @@
             </div>
         </header>
 
+        <!-- User Downloads Section -->
+        <section class="user-downloads-section" id="userDownloadsSection" style="display: none;">
+            <div class="form-container-unified">
+                <div class="form-card-unified">
+                    <h2>📥 My Downloads</h2>
+                    <p>Download your liked and saved posts from Reddit.</p>
+
+                    <!-- Download Mode Selection -->
+                    <div class="form-section">
+                        <h4>🎯 Download Mode</h4>
+                        <div class="radio-group mode-radio-group">
+                            <label class="radio-label mode-option" data-tooltip="Download media files (images, videos, gifs) from posts">
+                                <input type="radio" name="user_download_mode" value="download" checked>
+                                <span class="radio-custom"></span>
+                                <span class="mode-label">
+                                    <strong>Download</strong>
+                                    <small>Media files only</small>
+                                </span>
+                            </label>
+                            <label class="radio-label mode-option" data-tooltip="Save post metadata (title, author, comments) as JSON/XML without downloading media">
+                                <input type="radio" name="user_download_mode" value="archive">
+                                <span class="radio-custom"></span>
+                                <span class="mode-label">
+                                    <strong>Archive</strong>
+                                    <small>Metadata only</small>
+                                </span>
+                            </label>
+                            <label class="radio-label mode-option" data-tooltip="Download media files AND save metadata - complete backup of posts">
+                                <input type="radio" name="user_download_mode" value="clone">
+                                <span class="radio-custom"></span>
+                                <span class="mode-label">
+                                    <strong>Clone</strong>
+                                    <small>Media + Metadata</small>
+                                </span>
+                            </label>
+                        </div>
+                    </div>
+
+                    <!-- Scheduling Options -->
+                    <div class="form-section">
+                        <h4>⏰ Scheduling</h4>
+                        <div class="radio-group">
+                            <label class="radio-label">
+                                <input type="radio" name="user_schedule_type" value="now" checked>
+                                <span class="radio-custom"></span>
+                                Run Now
+                            </label>
+                            <label class="radio-label">
+                                <input type="radio" name="user_schedule_type" value="scheduled">
+                                <span class="radio-custom"></span>
+                                Schedule for Later
+                            </label>
+                        </div>
+
+                        <!-- Scheduled Options (shown when Schedule for Later is selected) -->
+                        <div id="userScheduleOptions" class="schedule-options" style="display: none;">
+                            <div class="form-group">
+                                <label for="userRunTime">Run Time (24-hour format):</label>
+                                <input type="time" id="userRunTime" name="user_run_time" value="02:00">
+                                <small class="form-help">Time to run the download daily (in your local timezone)</small>
+                            </div>
+                        </div>
+                    </div>
+
+                    <!-- Action Buttons -->
+                    <div class="form-section">
+                        <h4>🚀 Actions</h4>
+                        <div class="user-actions">
+                            <button id="downloadLikesBtn" class="btn btn-primary">❤️ Download My Likes</button>
+                            <button id="downloadSavedBtn" class="btn btn-primary">⭐ Download My Saved Posts</button>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </section>
+
         <main>
             <!-- Unified Download Form Section -->
             <section class="download-section">