diff --git a/bdfr/__main__.py b/bdfr/__main__.py index fd82055..4d12cad 100644 --- a/bdfr/__main__.py +++ b/bdfr/__main__.py @@ -26,6 +26,7 @@ _common_options = [ click.option("--file-scheme", default=None, type=str), click.option("--filename-restriction-scheme", type=click.Choice(("linux", "windows")), default=None), click.option("--folder-scheme", default=None, type=str), + click.option("--strip-unicode/--no-strip-unicode", default=None, help="Strip Unicode characters that cause Windows SMB issues (default: enabled)"), click.option("--ignore-user", type=str, multiple=True, default=None), click.option("--include-id-file", multiple=True, default=None), click.option("--log", type=str, default=None), diff --git a/bdfr/api.py b/bdfr/api.py index 4279bf9..f5dd528 100644 --- a/bdfr/api.py +++ b/bdfr/api.py @@ -1234,6 +1234,15 @@ class BDFRManager: config.saved = saved config.no_dupes = no_dupes config.simple_check = simple_check + + # Set authentication if token is available for this manager instance + if self.auth_token: + config.authenticate = True + config.auth_token = self.auth_token + logger.info(f"[DEBUG] Authentication enabled for user download with token") + else: + config.authenticate = False + logger.info(f"[DEBUG] No authentication token available for user download") download_id = self.create_download( DownloadType.USER, diff --git a/bdfr/configuration.py b/bdfr/configuration.py index 5811c0b..de373ac 100644 --- a/bdfr/configuration.py +++ b/bdfr/configuration.py @@ -25,6 +25,7 @@ class Configuration(Namespace): self.file_scheme: str = "{REDDITOR}_{TITLE}_{POSTID}" self.filename_restriction_scheme = None self.folder_scheme: str = "{SUBREDDIT}" + self.strip_unicode: bool = True self.ignore_user = [] self.include_id_file = [] self.limit: Optional[int] = None diff --git a/bdfr/connector.py b/bdfr/connector.py index 970c8c1..a32dd5c 100644 --- a/bdfr/connector.py +++ b/bdfr/connector.py @@ -441,7 +441,7 @@ class RedditConnector(metaclass=ABCMeta): def create_file_name_formatter(self) -> FileNameFormatter: return FileNameFormatter( - self.args.file_scheme, self.args.folder_scheme, self.args.time_format, self.args.filename_restriction_scheme + self.args.file_scheme, self.args.folder_scheme, self.args.time_format, self.args.filename_restriction_scheme, self.args.strip_unicode ) def create_time_filter(self) -> RedditTypes.TimeType: diff --git a/bdfr/file_name_formatter.py b/bdfr/file_name_formatter.py index aa59618..3df6829 100644 --- a/bdfr/file_name_formatter.py +++ b/bdfr/file_name_formatter.py @@ -36,6 +36,7 @@ class FileNameFormatter: directory_format_string: str, time_format_string: str, restriction_scheme: Optional[str] = None, + strip_unicode: bool = True, ): if not self.validate_string(file_format_string): raise BulkDownloaderException(f'"{file_format_string}" is not a valid format string') @@ -43,6 +44,7 @@ class FileNameFormatter: self.directory_format_string: list[str] = directory_format_string.split("/") self.time_format_string = time_format_string self.restiction_scheme = restriction_scheme.lower().strip() if restriction_scheme else None + self.strip_unicode = strip_unicode if self.restiction_scheme == "windows": self.max_path = self.WINDOWS_MAX_PATH_LENGTH else: @@ -65,12 +67,22 @@ class FileNameFormatter: result = result.replace("/", "") + # Strip Unicode characters that cause Windows SMB issues if enabled + if self.strip_unicode: + result = FileNameFormatter._strip_unicode_chars(result) + if self.restiction_scheme is None: if platform.system() == "Windows": result = FileNameFormatter._format_for_windows(result) + # Strip emojis on Windows if strip_unicode is enabled (for backward compatibility) + if self.strip_unicode: + result = FileNameFormatter._strip_emojis(result) elif self.restiction_scheme == "windows": logger.debug("Forcing Windows-compatible filenames") result = FileNameFormatter._format_for_windows(result) + # Strip emojis when forcing Windows compatibility if strip_unicode is enabled + if self.strip_unicode: + result = FileNameFormatter._strip_emojis(result) return result @staticmethod @@ -219,9 +231,33 @@ class FileNameFormatter: invalid_characters = r'<>:"\/|?*' for char in invalid_characters: input_string = input_string.replace(char, "") - input_string = FileNameFormatter._strip_emojis(input_string) return input_string + @staticmethod + def _strip_unicode_chars(input_string: str) -> str: + """Strip Unicode characters that cause Windows SMB to create 8.3 short names""" + import unicodedata + + # Remove emoji and symbols that cause Windows SMB issues + result = [] + for char in input_string: + # Keep ASCII characters + if ord(char) < 0x80: + result.append(char) + # Keep common Unicode letters, numbers, and punctuation + elif unicodedata.category(char) in ['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Ps', 'Pe', 'Pi', 'Pf', 'Po']: + result.append(char) + # Strip emoji, symbols, and other special characters that cause 8.3 names + elif unicodedata.category(char).startswith(('S', 'So', 'Sk', 'Sm')): # Symbols + continue + elif ord(char) > 0x1F000: # High Unicode ranges often contain emoji + continue + else: + # Keep other Unicode characters that are generally safe + result.append(char) + + return ''.join(result) + @staticmethod def _strip_emojis(input_string: str) -> str: result = input_string.encode("ascii", errors="ignore").decode("utf-8") diff --git a/data/bdfr-config/bdfr/default_config.cfg b/data/bdfr-config/bdfr/default_config.cfg new file mode 100644 index 0000000..986a4ba --- /dev/null +++ b/data/bdfr-config/bdfr/default_config.cfg @@ -0,0 +1,8 @@ +[DEFAULT] +client_id = U-6gk4ZCh3IeNQ +client_secret = 7CZHY6AmKweZME5s50SfDGylaPg +scopes = identity, history, read, save, mysubreddits +backup_log_count = 3 +max_wait_time = 120 +time_format = ISO + diff --git a/data/bdfr-config/default_config.cfg b/data/bdfr-config/default_config.cfg new file mode 100644 index 0000000..986a4ba --- /dev/null +++ b/data/bdfr-config/default_config.cfg @@ -0,0 +1,8 @@ +[DEFAULT] +client_id = U-6gk4ZCh3IeNQ +client_secret = 7CZHY6AmKweZME5s50SfDGylaPg +scopes = identity, history, read, save, mysubreddits +backup_log_count = 3 +max_wait_time = 120 +time_format = ISO + diff --git a/data/scheduled_tasks.db b/data/scheduled_tasks.db new file mode 100644 index 0000000..8be2102 Binary files /dev/null and b/data/scheduled_tasks.db differ diff --git a/data/scheduler_jobs.db b/data/scheduler_jobs.db new file mode 100644 index 0000000..651fe6e Binary files /dev/null and b/data/scheduler_jobs.db differ diff --git a/tests/test_file_name_formatter.py b/tests/test_file_name_formatter.py index f456415..23fee9b 100644 --- a/tests/test_file_name_formatter.py +++ b/tests/test_file_name_formatter.py @@ -519,3 +519,36 @@ def test_name_submission( results = test_formatter.format_resource_paths(test_resources, Path()) results = set([r[0].name for r in results]) assert results == expected_names + + +@pytest.mark.parametrize( + ("input_string", "expected"), + ( + ("Test 💕 emoji", "Test emoji"), + ("Normal text", "Normal text"), + ("Kirsty-Blue's post", "Kirsty-Blue's post"), + ("Hello 😀 world 🌍", "Hello world "), + ("No emoji here", "No emoji here"), + ), +) +def test_strip_unicode_chars(input_string: str, expected: str): + result = FileNameFormatter._strip_unicode_chars(input_string) + assert result == expected + + +def test_unicode_stripping_enabled(submission: MagicMock): + """Test that Unicode stripping is applied when enabled""" + submission.title = 'Test 💕 emoji' + formatter = FileNameFormatter("{TITLE}", "", "", strip_unicode=True) + result = formatter._format_name(submission, "{TITLE}") + assert "💕" not in result + assert result == "Test emoji" + + +def test_unicode_stripping_disabled(submission: MagicMock): + """Test that Unicode stripping is not applied when disabled""" + submission.title = 'Test 💕 emoji' + formatter = FileNameFormatter("{TITLE}", "", "", strip_unicode=False) + result = formatter._format_name(submission, "{TITLE}") + assert "💕" in result + assert result == "Test 💕 emoji" diff --git a/web_interface/app/main.py b/web_interface/app/main.py index 9741281..121d00d 100644 --- a/web_interface/app/main.py +++ b/web_interface/app/main.py @@ -527,6 +527,11 @@ async def create_download_with_bdfr_api(download_type: str, name: str, **kwargs) progress_callbacks=[callback] ) elif download_type == "user": + # Set auth token in the BDFR manager if provided + if auth_token: + bdfr_manager.auth_token = auth_token + logger.info(f"[DEBUG] Set auth token in BDFR manager for user download") + bdfr_download_id = bdfr_manager.download_user( name, limit=kwargs.get('limit'), diff --git a/web_interface/app/scheduled_tasks.py b/web_interface/app/scheduled_tasks.py index d1265ea..d59137c 100644 --- a/web_interface/app/scheduled_tasks.py +++ b/web_interface/app/scheduled_tasks.py @@ -486,15 +486,19 @@ async def get_queue_status(): ) +class UserDownloadRequest(BaseModel): + limit: int = Field(default=25, ge=1, le=1000) + sort: str = Field(default="hot") + download_mode: str = Field(default="download", pattern="^(download|archive|clone)$") + run_now: bool = Field(default=False) + run_time: str = Field(default="02:00", pattern="^([01]?[0-9]|2[0-3]):[0-5][0-9]$") + timezone: str = Field(default="UTC") + auth_state: Optional[str] = None + + @router.post("/create-likes", response_model=ScheduledTaskResponse, status_code=status.HTTP_201_CREATED) async def create_likes_task( - limit: int = 25, - sort: str = "hot", - download_mode: str = "download", - run_now: bool = False, - run_time: str = "02:00", - timezone: str = "UTC", - auth_state: str = None, + request: UserDownloadRequest, db: Session = Depends(get_db) ): """ @@ -502,14 +506,14 @@ async def create_likes_task( """ try: # Get current username from auth - if not auth_state: + if not request.auth_state: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Authentication required" ) oauth_manager = get_oauth_manager() - auth_status = oauth_manager.get_auth_status(auth_state) + auth_status = oauth_manager.get_auth_status(request.auth_state) if not auth_status["authenticated"]: raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, @@ -521,15 +525,15 @@ async def create_likes_task( # Validate timezone try: - pytz.timezone(timezone) + pytz.timezone(request.timezone) except pytz.exceptions.UnknownTimeZoneError: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Invalid timezone: {timezone}" + detail=f"Invalid timezone: {request.timezone}" ) # Parse run_time - hour, minute = map(int, run_time.split(':')) + hour, minute = map(int, request.run_time.split(':')) run_time_obj = time_type(hour=hour, minute=minute) # Create task @@ -538,16 +542,16 @@ async def create_likes_task( enabled=True, source_type="user", source_name=username, - download_mode=download_mode, - limit=limit, - sort=sort, + download_mode=request.download_mode, + limit=request.limit, + sort=request.sort, time_filter="day", no_dupes=True, simple_check=False, schedule_frequency="daily", run_time=run_time_obj, - timezone=timezone, - auth_state=auth_state, + timezone=request.timezone, + auth_state=request.auth_state, upvoted=True, saved=False ) @@ -564,7 +568,7 @@ async def create_likes_task( schedule_task(task) # If run_now is True, queue it immediately - if run_now: + if request.run_now: await task_queue.add_task(task.id, priority=1) logger.info(f"Created likes task {task.id}: {task.name}") @@ -584,13 +588,7 @@ async def create_likes_task( @router.post("/create-saved", response_model=ScheduledTaskResponse, status_code=status.HTTP_201_CREATED) async def create_saved_task( - limit: int = 25, - sort: str = "hot", - download_mode: str = "download", - run_now: bool = False, - run_time: str = "02:00", - timezone: str = "UTC", - auth_state: str = None, + request: UserDownloadRequest, db: Session = Depends(get_db) ): """ @@ -598,14 +596,14 @@ async def create_saved_task( """ try: # Get current username from auth - if not auth_state: + if not request.auth_state: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Authentication required" ) oauth_manager = get_oauth_manager() - auth_status = oauth_manager.get_auth_status(auth_state) + auth_status = oauth_manager.get_auth_status(request.auth_state) if not auth_status["authenticated"]: raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, @@ -617,15 +615,15 @@ async def create_saved_task( # Validate timezone try: - pytz.timezone(timezone) + pytz.timezone(request.timezone) except pytz.exceptions.UnknownTimeZoneError: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Invalid timezone: {timezone}" + detail=f"Invalid timezone: {request.timezone}" ) # Parse run_time - hour, minute = map(int, run_time.split(':')) + hour, minute = map(int, request.run_time.split(':')) run_time_obj = time_type(hour=hour, minute=minute) # Create task @@ -634,16 +632,16 @@ async def create_saved_task( enabled=True, source_type="user", source_name=username, - download_mode=download_mode, - limit=limit, - sort=sort, + download_mode=request.download_mode, + limit=request.limit, + sort=request.sort, time_filter="day", no_dupes=True, simple_check=False, schedule_frequency="daily", run_time=run_time_obj, - timezone=timezone, - auth_state=auth_state, + timezone=request.timezone, + auth_state=request.auth_state, upvoted=False, saved=True ) @@ -660,7 +658,7 @@ async def create_saved_task( schedule_task(task) # If run_now is True, queue it immediately - if run_now: + if request.run_now: await task_queue.add_task(task.id, priority=1) logger.info(f"Created saved task {task.id}: {task.name}") diff --git a/web_interface/app/scheduler.py b/web_interface/app/scheduler.py index cfe6cf3..d7887ca 100644 --- a/web_interface/app/scheduler.py +++ b/web_interface/app/scheduler.py @@ -231,8 +231,15 @@ async def execute_scheduled_task(task_id: str): # For user downloads, set submitted appropriately if task.source_type == 'user': - # For likes or saved, don't download submitted posts + # For likes or saved, still need authentication but don't download submitted posts + # Set submitted=False when downloading upvoted or saved posts kwargs['submitted'] = not (task.upvoted or task.saved) + + # Ensure auth_state is passed for user downloads requiring authentication + if task.upvoted or task.saved: + logger.info(f"User download task {task_id} requires authentication for {'likes' if task.upvoted else 'saved'} posts") + if not task.auth_state: + logger.warning(f"Task {task_id} needs authentication but no auth_state provided") # Create download using existing API download_id = await create_download_with_bdfr_api( diff --git a/web_interface/data/scheduled_tasks.db b/web_interface/data/scheduled_tasks.db index 5f8ad56..e32bcea 100644 Binary files a/web_interface/data/scheduled_tasks.db and b/web_interface/data/scheduled_tasks.db differ diff --git a/web_interface/data/scheduler_jobs.db b/web_interface/data/scheduler_jobs.db index 5f26353..20a6ac2 100644 Binary files a/web_interface/data/scheduler_jobs.db and b/web_interface/data/scheduler_jobs.db differ diff --git a/web_interface/templates/index.html b/web_interface/templates/index.html index 9c44b05..54f615e 100644 --- a/web_interface/templates/index.html +++ b/web_interface/templates/index.html @@ -1,11 +1,13 @@ + BDFR Web Interface +
@@ -22,89 +24,93 @@
- +
- -
@@ -115,7 +121,8 @@

🎯 Download Mode

-
@@ -320,10 +332,11 @@
-

© 2024 BDFR Web Interface. Powered by FastAPI.

+

© 2025 BDFR Web Interface. Powered by FastAPI.

+ \ No newline at end of file