fixes for file naming
formatting_check / formatting_check (push) Failing after 6s
Python Test / test (.sh, ubuntu-latest, 3.9) (push) Failing after 16s
Python Test / test (.ps1, windows-latest, 3.9) (push) Has been cancelled
Python Test / test (.sh, macos-latest, 3.9) (push) Has been cancelled

This commit is contained in:
2026-07-14 21:11:24 +12:00
parent 8f8e2c744d
commit f3dc8e46fd
16 changed files with 236 additions and 117 deletions
+1
View File
@@ -26,6 +26,7 @@ _common_options = [
click.option("--file-scheme", default=None, type=str),
click.option("--filename-restriction-scheme", type=click.Choice(("linux", "windows")), default=None),
click.option("--folder-scheme", default=None, type=str),
click.option("--strip-unicode/--no-strip-unicode", default=None, help="Strip Unicode characters that cause Windows SMB issues (default: enabled)"),
click.option("--ignore-user", type=str, multiple=True, default=None),
click.option("--include-id-file", multiple=True, default=None),
click.option("--log", type=str, default=None),
+9
View File
@@ -1235,6 +1235,15 @@ class BDFRManager:
config.no_dupes = no_dupes
config.simple_check = simple_check
# Set authentication if token is available for this manager instance
if self.auth_token:
config.authenticate = True
config.auth_token = self.auth_token
logger.info(f"[DEBUG] Authentication enabled for user download with token")
else:
config.authenticate = False
logger.info(f"[DEBUG] No authentication token available for user download")
download_id = self.create_download(
DownloadType.USER,
username,
+1
View File
@@ -25,6 +25,7 @@ class Configuration(Namespace):
self.file_scheme: str = "{REDDITOR}_{TITLE}_{POSTID}"
self.filename_restriction_scheme = None
self.folder_scheme: str = "{SUBREDDIT}"
self.strip_unicode: bool = True
self.ignore_user = []
self.include_id_file = []
self.limit: Optional[int] = None
+1 -1
View File
@@ -441,7 +441,7 @@ class RedditConnector(metaclass=ABCMeta):
def create_file_name_formatter(self) -> FileNameFormatter:
return FileNameFormatter(
self.args.file_scheme, self.args.folder_scheme, self.args.time_format, self.args.filename_restriction_scheme
self.args.file_scheme, self.args.folder_scheme, self.args.time_format, self.args.filename_restriction_scheme, self.args.strip_unicode
)
def create_time_filter(self) -> RedditTypes.TimeType:
+37 -1
View File
@@ -36,6 +36,7 @@ class FileNameFormatter:
directory_format_string: str,
time_format_string: str,
restriction_scheme: Optional[str] = None,
strip_unicode: bool = True,
):
if not self.validate_string(file_format_string):
raise BulkDownloaderException(f'"{file_format_string}" is not a valid format string')
@@ -43,6 +44,7 @@ class FileNameFormatter:
self.directory_format_string: list[str] = directory_format_string.split("/")
self.time_format_string = time_format_string
self.restiction_scheme = restriction_scheme.lower().strip() if restriction_scheme else None
self.strip_unicode = strip_unicode
if self.restiction_scheme == "windows":
self.max_path = self.WINDOWS_MAX_PATH_LENGTH
else:
@@ -65,12 +67,22 @@ class FileNameFormatter:
result = result.replace("/", "")
# Strip Unicode characters that cause Windows SMB issues if enabled
if self.strip_unicode:
result = FileNameFormatter._strip_unicode_chars(result)
if self.restiction_scheme is None:
if platform.system() == "Windows":
result = FileNameFormatter._format_for_windows(result)
# Strip emojis on Windows if strip_unicode is enabled (for backward compatibility)
if self.strip_unicode:
result = FileNameFormatter._strip_emojis(result)
elif self.restiction_scheme == "windows":
logger.debug("Forcing Windows-compatible filenames")
result = FileNameFormatter._format_for_windows(result)
# Strip emojis when forcing Windows compatibility if strip_unicode is enabled
if self.strip_unicode:
result = FileNameFormatter._strip_emojis(result)
return result
@staticmethod
@@ -219,9 +231,33 @@ class FileNameFormatter:
invalid_characters = r'<>:"\/|?*'
for char in invalid_characters:
input_string = input_string.replace(char, "")
input_string = FileNameFormatter._strip_emojis(input_string)
return input_string
@staticmethod
def _strip_unicode_chars(input_string: str) -> str:
"""Strip Unicode characters that cause Windows SMB to create 8.3 short names"""
import unicodedata
# Remove emoji and symbols that cause Windows SMB issues
result = []
for char in input_string:
# Keep ASCII characters
if ord(char) < 0x80:
result.append(char)
# Keep common Unicode letters, numbers, and punctuation
elif unicodedata.category(char) in ['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Ps', 'Pe', 'Pi', 'Pf', 'Po']:
result.append(char)
# Strip emoji, symbols, and other special characters that cause 8.3 names
elif unicodedata.category(char).startswith(('S', 'So', 'Sk', 'Sm')): # Symbols
continue
elif ord(char) > 0x1F000: # High Unicode ranges often contain emoji
continue
else:
# Keep other Unicode characters that are generally safe
result.append(char)
return ''.join(result)
@staticmethod
def _strip_emojis(input_string: str) -> str:
result = input_string.encode("ascii", errors="ignore").decode("utf-8")
+8
View File
@@ -0,0 +1,8 @@
[DEFAULT]
client_id = U-6gk4ZCh3IeNQ
client_secret = 7CZHY6AmKweZME5s50SfDGylaPg
scopes = identity, history, read, save, mysubreddits
backup_log_count = 3
max_wait_time = 120
time_format = ISO
+8
View File
@@ -0,0 +1,8 @@
[DEFAULT]
client_id = U-6gk4ZCh3IeNQ
client_secret = 7CZHY6AmKweZME5s50SfDGylaPg
scopes = identity, history, read, save, mysubreddits
backup_log_count = 3
max_wait_time = 120
time_format = ISO
Binary file not shown.
Binary file not shown.
+33
View File
@@ -519,3 +519,36 @@ def test_name_submission(
results = test_formatter.format_resource_paths(test_resources, Path())
results = set([r[0].name for r in results])
assert results == expected_names
@pytest.mark.parametrize(
("input_string", "expected"),
(
("Test 💕 emoji", "Test emoji"),
("Normal text", "Normal text"),
("Kirsty-Blue's post", "Kirsty-Blue's post"),
("Hello 😀 world 🌍", "Hello world "),
("No emoji here", "No emoji here"),
),
)
def test_strip_unicode_chars(input_string: str, expected: str):
result = FileNameFormatter._strip_unicode_chars(input_string)
assert result == expected
def test_unicode_stripping_enabled(submission: MagicMock):
"""Test that Unicode stripping is applied when enabled"""
submission.title = 'Test 💕 emoji'
formatter = FileNameFormatter("{TITLE}", "", "", strip_unicode=True)
result = formatter._format_name(submission, "{TITLE}")
assert "💕" not in result
assert result == "Test emoji"
def test_unicode_stripping_disabled(submission: MagicMock):
"""Test that Unicode stripping is not applied when disabled"""
submission.title = 'Test 💕 emoji'
formatter = FileNameFormatter("{TITLE}", "", "", strip_unicode=False)
result = formatter._format_name(submission, "{TITLE}")
assert "💕" in result
assert result == "Test 💕 emoji"
+5
View File
@@ -527,6 +527,11 @@ async def create_download_with_bdfr_api(download_type: str, name: str, **kwargs)
progress_callbacks=[callback]
)
elif download_type == "user":
# Set auth token in the BDFR manager if provided
if auth_token:
bdfr_manager.auth_token = auth_token
logger.info(f"[DEBUG] Set auth token in BDFR manager for user download")
bdfr_download_id = bdfr_manager.download_user(
name,
limit=kwargs.get('limit'),
+34 -36
View File
@@ -486,15 +486,19 @@ async def get_queue_status():
)
class UserDownloadRequest(BaseModel):
limit: int = Field(default=25, ge=1, le=1000)
sort: str = Field(default="hot")
download_mode: str = Field(default="download", pattern="^(download|archive|clone)$")
run_now: bool = Field(default=False)
run_time: str = Field(default="02:00", pattern="^([01]?[0-9]|2[0-3]):[0-5][0-9]$")
timezone: str = Field(default="UTC")
auth_state: Optional[str] = None
@router.post("/create-likes", response_model=ScheduledTaskResponse, status_code=status.HTTP_201_CREATED)
async def create_likes_task(
limit: int = 25,
sort: str = "hot",
download_mode: str = "download",
run_now: bool = False,
run_time: str = "02:00",
timezone: str = "UTC",
auth_state: str = None,
request: UserDownloadRequest,
db: Session = Depends(get_db)
):
"""
@@ -502,14 +506,14 @@ async def create_likes_task(
"""
try:
# Get current username from auth
if not auth_state:
if not request.auth_state:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Authentication required"
)
oauth_manager = get_oauth_manager()
auth_status = oauth_manager.get_auth_status(auth_state)
auth_status = oauth_manager.get_auth_status(request.auth_state)
if not auth_status["authenticated"]:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
@@ -521,15 +525,15 @@ async def create_likes_task(
# Validate timezone
try:
pytz.timezone(timezone)
pytz.timezone(request.timezone)
except pytz.exceptions.UnknownTimeZoneError:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Invalid timezone: {timezone}"
detail=f"Invalid timezone: {request.timezone}"
)
# Parse run_time
hour, minute = map(int, run_time.split(':'))
hour, minute = map(int, request.run_time.split(':'))
run_time_obj = time_type(hour=hour, minute=minute)
# Create task
@@ -538,16 +542,16 @@ async def create_likes_task(
enabled=True,
source_type="user",
source_name=username,
download_mode=download_mode,
limit=limit,
sort=sort,
download_mode=request.download_mode,
limit=request.limit,
sort=request.sort,
time_filter="day",
no_dupes=True,
simple_check=False,
schedule_frequency="daily",
run_time=run_time_obj,
timezone=timezone,
auth_state=auth_state,
timezone=request.timezone,
auth_state=request.auth_state,
upvoted=True,
saved=False
)
@@ -564,7 +568,7 @@ async def create_likes_task(
schedule_task(task)
# If run_now is True, queue it immediately
if run_now:
if request.run_now:
await task_queue.add_task(task.id, priority=1)
logger.info(f"Created likes task {task.id}: {task.name}")
@@ -584,13 +588,7 @@ async def create_likes_task(
@router.post("/create-saved", response_model=ScheduledTaskResponse, status_code=status.HTTP_201_CREATED)
async def create_saved_task(
limit: int = 25,
sort: str = "hot",
download_mode: str = "download",
run_now: bool = False,
run_time: str = "02:00",
timezone: str = "UTC",
auth_state: str = None,
request: UserDownloadRequest,
db: Session = Depends(get_db)
):
"""
@@ -598,14 +596,14 @@ async def create_saved_task(
"""
try:
# Get current username from auth
if not auth_state:
if not request.auth_state:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Authentication required"
)
oauth_manager = get_oauth_manager()
auth_status = oauth_manager.get_auth_status(auth_state)
auth_status = oauth_manager.get_auth_status(request.auth_state)
if not auth_status["authenticated"]:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
@@ -617,15 +615,15 @@ async def create_saved_task(
# Validate timezone
try:
pytz.timezone(timezone)
pytz.timezone(request.timezone)
except pytz.exceptions.UnknownTimeZoneError:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Invalid timezone: {timezone}"
detail=f"Invalid timezone: {request.timezone}"
)
# Parse run_time
hour, minute = map(int, run_time.split(':'))
hour, minute = map(int, request.run_time.split(':'))
run_time_obj = time_type(hour=hour, minute=minute)
# Create task
@@ -634,16 +632,16 @@ async def create_saved_task(
enabled=True,
source_type="user",
source_name=username,
download_mode=download_mode,
limit=limit,
sort=sort,
download_mode=request.download_mode,
limit=request.limit,
sort=request.sort,
time_filter="day",
no_dupes=True,
simple_check=False,
schedule_frequency="daily",
run_time=run_time_obj,
timezone=timezone,
auth_state=auth_state,
timezone=request.timezone,
auth_state=request.auth_state,
upvoted=False,
saved=True
)
@@ -660,7 +658,7 @@ async def create_saved_task(
schedule_task(task)
# If run_now is True, queue it immediately
if run_now:
if request.run_now:
await task_queue.add_task(task.id, priority=1)
logger.info(f"Created saved task {task.id}: {task.name}")
+8 -1
View File
@@ -231,9 +231,16 @@ async def execute_scheduled_task(task_id: str):
# For user downloads, set submitted appropriately
if task.source_type == 'user':
# For likes or saved, don't download submitted posts
# For likes or saved, still need authentication but don't download submitted posts
# Set submitted=False when downloading upvoted or saved posts
kwargs['submitted'] = not (task.upvoted or task.saved)
# Ensure auth_state is passed for user downloads requiring authentication
if task.upvoted or task.saved:
logger.info(f"User download task {task_id} requires authentication for {'likes' if task.upvoted else 'saved'} posts")
if not task.auth_state:
logger.warning(f"Task {task_id} needs authentication but no auth_state provided")
# Create download using existing API
download_id = await create_download_with_bdfr_api(
download_type=task.source_type,
Binary file not shown.
Binary file not shown.
+91 -78
View File
@@ -1,11 +1,13 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>BDFR Web Interface</title>
<link rel="stylesheet" href="/static/css/style.css">
</head>
<body>
<div class="container">
<header>
@@ -22,89 +24,93 @@
</div>
<div class="auth-actions">
<button id="loginBtn" class="btn btn-small btn-outline">🔐 Login with Reddit</button>
<button id="logoutBtn" class="btn btn-small btn-outline" style="display: none;">🚪 Logout</button>
<button id="logoutBtn" class="btn btn-small btn-outline" style="display: none;">🚪
Logout</button>
</div>
</div>
</div>
</header>
<!-- User Downloads Section -->
<section class="user-downloads-section" id="userDownloadsSection" style="display: none;">
<div class="form-container-unified">
<div class="form-card-unified">
<h2>📥 My Downloads</h2>
<p>Download your liked and saved posts from Reddit.</p>
<main>
<!-- User Downloads Section -->
<section class="user-downloads-section" id="userDownloadsSection" style="display: none;">
<div class="form-container-unified">
<div class="form-card-unified">
<h2>📥 My Downloads</h2>
<p>Download your liked and saved posts from Reddit.</p>
<!-- Download Mode Selection -->
<div class="form-section">
<h4>🎯 Download Mode</h4>
<div class="radio-group mode-radio-group">
<label class="radio-label mode-option" data-tooltip="Download media files (images, videos, gifs) from posts">
<input type="radio" name="user_download_mode" value="download" checked>
<span class="radio-custom"></span>
<span class="mode-label">
<strong>Download</strong>
<small>Media files only</small>
</span>
</label>
<label class="radio-label mode-option" data-tooltip="Save post metadata (title, author, comments) as JSON/XML without downloading media">
<input type="radio" name="user_download_mode" value="archive">
<span class="radio-custom"></span>
<span class="mode-label">
<strong>Archive</strong>
<small>Metadata only</small>
</span>
</label>
<label class="radio-label mode-option" data-tooltip="Download media files AND save metadata - complete backup of posts">
<input type="radio" name="user_download_mode" value="clone">
<span class="radio-custom"></span>
<span class="mode-label">
<strong>Clone</strong>
<small>Media + Metadata</small>
</span>
</label>
</div>
</div>
<!-- Scheduling Options -->
<div class="form-section">
<h4>⏰ Scheduling</h4>
<div class="radio-group">
<label class="radio-label">
<input type="radio" name="user_schedule_type" value="now" checked>
<span class="radio-custom"></span>
Run Now
</label>
<label class="radio-label">
<input type="radio" name="user_schedule_type" value="scheduled">
<span class="radio-custom"></span>
Schedule for Later
</label>
<!-- Download Mode Selection -->
<div class="form-section">
<h4>🎯 Download Mode</h4>
<div class="radio-group mode-radio-group">
<label class="radio-label mode-option"
data-tooltip="Download media files (images, videos, gifs) from posts">
<input type="radio" name="user_download_mode" value="download" checked>
<span class="radio-custom"></span>
<span class="mode-label">
<strong>Download</strong>
<small>Media files only</small>
</span>
</label>
<label class="radio-label mode-option"
data-tooltip="Save post metadata (title, author, comments) as JSON/XML without downloading media">
<input type="radio" name="user_download_mode" value="archive">
<span class="radio-custom"></span>
<span class="mode-label">
<strong>Archive</strong>
<small>Metadata only</small>
</span>
</label>
<label class="radio-label mode-option"
data-tooltip="Download media files AND save metadata - complete backup of posts">
<input type="radio" name="user_download_mode" value="clone">
<span class="radio-custom"></span>
<span class="mode-label">
<strong>Clone</strong>
<small>Media + Metadata</small>
</span>
</label>
</div>
</div>
<!-- Scheduled Options (shown when Schedule for Later is selected) -->
<div id="userScheduleOptions" class="schedule-options" style="display: none;">
<div class="form-group">
<label for="userRunTime">Run Time (24-hour format):</label>
<input type="time" id="userRunTime" name="user_run_time" value="02:00">
<small class="form-help">Time to run the download daily (in your local timezone)</small>
<!-- Scheduling Options -->
<div class="form-section">
<h4>⏰ Scheduling</h4>
<div class="radio-group">
<label class="radio-label">
<input type="radio" name="user_schedule_type" value="now" checked>
<span class="radio-custom"></span>
Run Now
</label>
<label class="radio-label">
<input type="radio" name="user_schedule_type" value="scheduled">
<span class="radio-custom"></span>
Schedule for Later
</label>
</div>
<!-- Scheduled Options (shown when Schedule for Later is selected) -->
<div id="userScheduleOptions" class="schedule-options" style="display: none;">
<div class="form-group">
<label for="userRunTime">Run Time (24-hour format):</label>
<input type="time" id="userRunTime" name="user_run_time" value="02:00">
<small class="form-help">Time to run the download daily (in your local
timezone)</small>
</div>
</div>
</div>
<!-- Action Buttons -->
<div class="form-section">
<h4>🚀 Actions</h4>
<div class="user-actions">
<button id="downloadLikesBtn" class="btn btn-primary">❤️ Download My Likes</button>
<button id="downloadSavedBtn" class="btn btn-primary">⭐ Download My Saved Posts</button>
</div>
</div>
</div>
<!-- Action Buttons -->
<div class="form-section">
<h4>🚀 Actions</h4>
<div class="user-actions">
<button id="downloadLikesBtn" class="btn btn-primary">❤️ Download My Likes</button>
<button id="downloadSavedBtn" class="btn btn-primary">⭐ Download My Saved Posts</button>
</div>
</div>
</div>
</div>
</section>
<main>
</section>
<!-- Unified Download Form Section -->
<section class="download-section">
<div class="form-container-unified">
@@ -115,7 +121,8 @@
<div class="form-section mode-section">
<h4>🎯 Download Mode</h4>
<div class="radio-group mode-radio-group">
<label class="radio-label mode-option" data-tooltip="Download media files (images, videos, gifs) from posts">
<label class="radio-label mode-option"
data-tooltip="Download media files (images, videos, gifs) from posts">
<input type="radio" name="download_mode" value="download" checked>
<span class="radio-custom"></span>
<span class="mode-label">
@@ -123,7 +130,8 @@
<small>Media files only</small>
</span>
</label>
<label class="radio-label mode-option" data-tooltip="Save post metadata (title, author, comments) as JSON/XML without downloading media">
<label class="radio-label mode-option"
data-tooltip="Save post metadata (title, author, comments) as JSON/XML without downloading media">
<input type="radio" name="download_mode" value="archive">
<span class="radio-custom"></span>
<span class="mode-label">
@@ -131,7 +139,8 @@
<small>Metadata only</small>
</span>
</label>
<label class="radio-label mode-option" data-tooltip="Download media files AND save metadata - complete backup of posts">
<label class="radio-label mode-option"
data-tooltip="Download media files AND save metadata - complete backup of posts">
<input type="radio" name="download_mode" value="clone">
<span class="radio-custom"></span>
<span class="mode-label">
@@ -162,7 +171,8 @@
<!-- Source Name Input -->
<div class="form-group">
<label for="sourceName" id="sourceNameLabel">Subreddit Name:</label>
<input type="text" id="sourceName" name="source_name" placeholder="e.g., python, machinelearning" required>
<input type="text" id="sourceName" name="source_name"
placeholder="e.g., python, machinelearning" required>
<small class="form-help" id="sourceNameHelp">Enter subreddit name without 'r/'</small>
</div>
@@ -239,13 +249,15 @@
<div id="scheduleOptions" class="schedule-options" style="display: none;">
<div class="form-group">
<label for="taskName">Task Name:</label>
<input type="text" id="taskName" name="task_name" placeholder="e.g., Daily Python Posts">
<input type="text" id="taskName" name="task_name"
placeholder="e.g., Daily Python Posts">
<small class="form-help">A friendly name to identify this scheduled task</small>
</div>
<div class="form-group">
<label for="runTime">Run Time (24-hour format):</label>
<input type="time" id="runTime" name="run_time" value="02:00">
<small class="form-help">Time to run the download daily (in your local timezone)</small>
<small class="form-help">Time to run the download daily (in your local
timezone)</small>
</div>
</div>
</div>
@@ -320,10 +332,11 @@
</main>
<footer>
<p>&copy; 2024 BDFR Web Interface. Powered by FastAPI.</p>
<p>&copy; 2025 BDFR Web Interface. Powered by FastAPI.</p>
</footer>
</div>
<script src="/static/js/app.js"></script>
</body>
</html>