ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reformatting
formatting_check / formatting_check (push) Failing after 3s
Details
Python Test / test (.ps1, windows-latest, 3.9) (push) Has been cancelled
Details
Python Test / test (.sh, macos-latest, 3.9) (push) Has been cancelled
Details
Python Test / test (.sh, ubuntu-latest, 3.9) (push) Has been cancelled
Details

Browse Source
This commit is contained in:
ModerateWinGuy
2026-07-14 21:32:55 +12:00
parent 3d0658b483
commit 0157f462cc
17 changed files with 439 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bdfr/__main__.py
+11 -2
View File
@@ -26,7 +26,11 @@ _common_options = [
click.option("--file-scheme", default=None, type=str),
click.option("--filename-restriction-scheme", type=click.Choice(("linux", "windows")), default=None),
click.option("--folder-scheme", default=None, type=str),
click.option("--strip-unicode/--no-strip-unicode", default=None, help="Strip Unicode characters that cause Windows SMB issues (default: enabled)"),
click.option(
"--strip-unicode/--no-strip-unicode",
default=None,
help="Strip Unicode characters that cause Windows SMB issues (default: enabled)",
),
click.option("--ignore-user", type=str, multiple=True, default=None),
click.option("--include-id-file", multiple=True, default=None),
click.option("--log", type=str, default=None),
@@ -54,7 +58,12 @@ _downloader_options = [
click.option("--max-wait-time", type=int, default=None),
click.option("--no-dupes", is_flag=True, default=None),
click.option("--search-existing", is_flag=True, default=None),
click.option("--simple-check", is_flag=True, default=None, help="Enable fast URL-based duplicate checking (works with --no-dupes)"),
click.option(
"--simple-check",
is_flag=True,
default=None,
help="Enable fast URL-based duplicate checking (works with --no-dupes)",
),
click.option("--skip", default=None, multiple=True),
click.option("--skip-domain", default=None, multiple=True),
click.option("--skip-subreddit", default=None, multiple=True),
bdfr/api.py
+194 -153
View File
@@ -25,12 +25,12 @@ from typing import Any, Dict, List, Optional, Union
import prawcore
from bdfr import exceptions as errors
from bdfr.archiver import Archiver
from bdfr.cloner import RedditCloner
from bdfr.configuration import Configuration
from bdfr.connector import RedditConnector
from bdfr.downloader import RedditDownloader
from bdfr.archiver import Archiver
from bdfr.cloner import RedditCloner
from bdfr import exceptions as errors
logger = logging.getLogger(__name__)
@@ -84,7 +84,9 @@ def retry_reddit_api_call(api_call_func: Callable, max_retries: int = 5, base_wa
raise Exception(f"Reddit API rate limited after {max_retries + 1} attempts: {e}")
# Wait before retrying
logger.info(f"Reddit API rate limited, waiting {current_wait_time} seconds before retry {attempt + 1}/{max_retries}")
logger.info(
f"Reddit API rate limited, waiting {current_wait_time} seconds before retry {attempt + 1}/{max_retries}"
)
time.sleep(current_wait_time)
# Increase wait time for next attempt (exponential backoff)
@@ -93,6 +95,7 @@ def retry_reddit_api_call(api_call_func: Callable, max_retries: int = 5, base_wa
class DownloadType(Enum):
"""Types of downloads supported by BDFR"""
SUBREDDIT = "subreddit"
USER = "user"
MULTIREDDIT = "multireddit"
@@ -103,6 +106,7 @@ class DownloadType(Enum):
class DownloadStatus(Enum):
"""Status of a download operation"""
QUEUED = "queued"
RUNNING = "running"
COMPLETED = "completed"
@@ -121,7 +125,7 @@ class ProgressEvent:
message: str,
progress: Optional[float] = None,
data: Optional[Dict[str, Any]] = None,
timestamp: Optional[datetime] = None
timestamp: Optional[datetime] = None,
):
self.event_type = event_type # "progress", "status", "error", "completed"
self.download_id = download_id
@@ -138,7 +142,7 @@ class ProgressEvent:
"message": self.message,
"progress": self.progress,
"data": self.data,
"timestamp": self.timestamp.isoformat()
"timestamp": self.timestamp.isoformat(),
}
@@ -177,8 +181,8 @@ class LoggingCallback(ProgressCallback):
async def on_error(self, event: ProgressEvent):
"""Log error events"""
self.logger.error(f"[{event.download_id}] ERROR: {event.message}")
if event.data.get('exception'):
self.logger.exception(f"[{event.download_id}] Exception details", exc_info=event.data['exception'])
if event.data.get("exception"):
self.logger.exception(f"[{event.download_id}] Exception details", exc_info=event.data["exception"])
async def on_completed(self, event: ProgressEvent):
"""Log completion events"""
@@ -258,7 +262,7 @@ class BDFRManager:
download_type: DownloadType,
name: str,
config: Optional[Configuration] = None,
progress_callbacks: Optional[List[ProgressCallback]] = None
progress_callbacks: Optional[List[ProgressCallback]] = None,
) -> str:
"""
Create a new download operation.
@@ -326,7 +330,7 @@ class BDFRManager:
"items_processed": 0,
"items_found": 0,
"current_item": None,
"phase": "queued"
"phase": "queued",
}
self.logger.info(f"Created download {download_id} for {download_type.value}: {name}")
@@ -356,11 +360,7 @@ class BDFRManager:
download_info["start_time"] = datetime.now()
# Create and start download thread
download_thread = threading.Thread(
target=self._run_download,
args=(download_id,),
daemon=True
)
download_thread = threading.Thread(target=self._run_download, args=(download_id,), daemon=True)
self._download_threads[download_id] = download_thread
download_thread.start()
@@ -375,8 +375,12 @@ class BDFRManager:
downloader = None
try:
self.logger.info(f"[DEBUG] Starting download {download_id} for {download_info['type']}: {download_info['name']}")
self.logger.info(f"[DEBUG] Config: subreddit={getattr(config, 'subreddit', None)}, user={getattr(config, 'user', None)}, limit={getattr(config, 'limit', None)}")
self.logger.info(
f"[DEBUG] Starting download {download_id} for {download_info['type']}: {download_info['name']}"
)
self.logger.info(
f"[DEBUG] Config: subreddit={getattr(config, 'subreddit', None)}, user={getattr(config, 'user', None)}, limit={getattr(config, 'limit', None)}"
)
# Create progress event for start
start_event = ProgressEvent(
@@ -384,11 +388,13 @@ class BDFRManager:
download_id,
f"Starting {download_info['type']} download: {download_info['name']}",
0.0,
{"phase": "starting"}
{"phase": "starting"},
)
# Notify callbacks
self.logger.info(f"[BDFR-API] Notifying {len(callbacks)} callbacks of progress event for download {download_id}")
self.logger.info(
f"[BDFR-API] Notifying {len(callbacks)} callbacks of progress event for download {download_id}"
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", start_event))
# Set up authentication token in config if provided
@@ -411,7 +417,9 @@ class BDFRManager:
log_path.parent.mkdir(parents=True, exist_ok=True)
# Determine which BDFR class to use based on configuration
self.logger.info(f"[DEBUG] Determining downloader type - submitted={getattr(config, 'submitted', False)}, upvoted={getattr(config, 'upvoted', False)}, saved={getattr(config, 'saved', False)}, format={getattr(config, 'format', 'N/A')}")
self.logger.info(
f"[DEBUG] Determining downloader type - submitted={getattr(config, 'submitted', False)}, upvoted={getattr(config, 'upvoted', False)}, saved={getattr(config, 'saved', False)}, format={getattr(config, 'format', 'N/A')}"
)
try:
if config.submitted or config.upvoted or config.saved:
@@ -420,7 +428,7 @@ class BDFRManager:
downloader = RedditDownloader(config, [])
download_info["operation_type"] = "user_download"
self.logger.info(f"[DEBUG] Using RedditDownloader for user data")
elif hasattr(config, 'format') and config.format in ["json", "xml", "yaml"]:
elif hasattr(config, "format") and config.format in ["json", "xml", "yaml"]:
# Archive operation
self.logger.info(f"[DEBUG] Creating Archiver for archive operation")
downloader = Archiver(config, [])
@@ -447,7 +455,9 @@ class BDFRManager:
# Mark as completed only if not already failed or cancelled
if download_info.get("status") not in [DownloadStatus.FAILED.value, DownloadStatus.CANCELLED.value]:
self.logger.info(f"[DEBUG] Marking download {download_id} as completed")
self.logger.info(f"[DEBUG] Final stats - items_processed: {download_info['items_processed']}, items_found: {download_info['items_found']}")
self.logger.info(
f"[DEBUG] Final stats - items_processed: {download_info['items_processed']}, items_found: {download_info['items_found']}"
)
download_info["status"] = DownloadStatus.COMPLETED.value
download_info["progress"] = 100.0
download_info["end_time"] = datetime.now()
@@ -460,18 +470,21 @@ class BDFRManager:
{
"items_processed": download_info["items_processed"],
"items_found": download_info["items_found"],
"operation_type": download_info.get("operation_type")
}
"operation_type": download_info.get("operation_type"),
},
)
self.logger.info(f"[DEBUG] Sending completion callback for {download_id}")
self.logger.info(f"[BDFR-API] Sending completion event for download {download_id}")
asyncio.run(self._notify_callbacks(callbacks, "on_completed", completed_event))
else:
self.logger.info(f"[DEBUG] Skipping completion for {download_id} due to status={download_info.get('status')}")
self.logger.info(
f"[DEBUG] Skipping completion for {download_id} due to status={download_info.get('status')}"
)
except Exception as e:
import traceback
# Get full stack trace for debugging
stack_trace = traceback.format_exc()
@@ -479,6 +492,7 @@ class BDFRManager:
is_rate_limited = False
try:
import prawcore
TooManyRequests = getattr(prawcore.exceptions, "TooManyRequests", None)
if TooManyRequests is not None and isinstance(e, TooManyRequests):
is_rate_limited = True
@@ -503,7 +517,11 @@ class BDFRManager:
download_id,
f"Download failed: {str(e)}",
None,
{"exception": str(e), "stack_trace": stack_trace, "phase": ("rate_limited" if is_rate_limited else "failed")}
{
"exception": str(e),
"stack_trace": stack_trace,
"phase": ("rate_limited" if is_rate_limited else "failed"),
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
@@ -519,17 +537,19 @@ class BDFRManager:
# Close any logging handlers that might be holding file locks
logger = logging.getLogger()
for handler in logger.handlers[:]:
if hasattr(handler, 'close') and hasattr(handler, 'baseFilename'):
if hasattr(handler, "close") and hasattr(handler, "baseFilename"):
# Check if this handler is for our download's log file
if hasattr(config, 'log') and config.log:
handler_path = getattr(handler, 'baseFilename', '')
if hasattr(config, "log") and config.log:
handler_path = getattr(handler, "baseFilename", "")
if handler_path == config.log or handler_path.startswith(str(self.download_directory)):
try:
handler.close()
logger.removeHandler(handler)
self.logger.info(f"[DEBUG] Closed logging handler for {handler_path}")
except Exception as cleanup_error:
self.logger.warning(f"[DEBUG] Error closing handler {handler_path}: {cleanup_error}")
self.logger.warning(
f"[DEBUG] Error closing handler {handler_path}: {cleanup_error}"
)
except Exception as cleanup_error:
self.logger.warning(f"[DEBUG] Error during logging cleanup: {cleanup_error}")
@@ -549,17 +569,13 @@ class BDFRManager:
# Update phase to fetching
download_info["phase"] = "fetching_submissions"
progress_event = ProgressEvent(
"progress",
download_id,
"Fetching submissions...",
10.0,
{"phase": "fetching_submissions"}
"progress", download_id, "Fetching submissions...", 10.0, {"phase": "fetching_submissions"}
)
self.logger.info(f"[BDFR-API] Sending progress update for download {download_id}")
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
# For RedditDownloader, we need to intercept the download process
if hasattr(downloader, 'reddit_lists') and hasattr(downloader, '_download_submission'):
if hasattr(downloader, "reddit_lists") and hasattr(downloader, "_download_submission"):
processed_submissions = 0
self.logger.info(f"[DEBUG] RedditDownloader has reddit_lists and _download_submission methods")
@@ -567,7 +583,7 @@ class BDFRManager:
# Get the limit from config - this is our target/goal for items_found
# items_found represents "how many items are available to download (up to limit)"
# items_processed represents "how many we've actually processed so far"
limit = getattr(config, 'limit', 100)
limit = getattr(config, "limit", 100)
download_info["items_found"] = limit
self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}")
@@ -590,7 +606,9 @@ class BDFRManager:
return
try:
self.logger.info(f"[DEBUG] Processing submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}")
self.logger.info(
f"[DEBUG] Processing submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}"
)
# Update current item being processed
download_info["current_item"] = submission.id
@@ -614,8 +632,8 @@ class BDFRManager:
"current_item_type": "submission",
"current_subreddit": submission.subreddit.display_name,
"items_processed": processed_submissions,
"items_found": download_info["items_found"] # Keep showing the limit as target
}
"items_found": download_info["items_found"], # Keep showing the limit as target
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
@@ -628,8 +646,12 @@ class BDFRManager:
# Check if this is a rate limiting error that should be retried
error_msg = str(e)
if "429" in error_msg:
self.logger.warning(f"[DEBUG] Rate limited while processing submission {submission.id}: {e}")
self.logger.warning(f"[DEBUG] Will continue with next submission instead of failing entire download")
self.logger.warning(
f"[DEBUG] Rate limited while processing submission {submission.id}: {e}"
)
self.logger.warning(
f"[DEBUG] Will continue with next submission instead of failing entire download"
)
continue
else:
# Not a rate limiting error, re-raise
@@ -640,6 +662,7 @@ class BDFRManager:
except Exception as e:
import traceback
error_msg = str(e)
submission_id = submission.id if submission is not None else "unknown"
@@ -647,7 +670,9 @@ class BDFRManager:
stack_trace = traceback.format_exc()
if "429" in error_msg:
self.logger.error(f"[DEBUG] Rate limited while processing submission {submission_id}: {e}")
self.logger.error(
f"[DEBUG] Rate limited while processing submission {submission_id}: {e}"
)
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
# Mark download as failed due to rate limiting
download_info["status"] = DownloadStatus.FAILED.value
@@ -659,7 +684,7 @@ class BDFRManager:
download_id,
f"Rate limited by Reddit API: {e}",
None,
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"}
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"},
)
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
return
@@ -675,9 +700,11 @@ class BDFRManager:
if actual_count < limit:
download_info["items_found"] = actual_count
self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}")
self.logger.info(f"[DEBUG] Final progress update - processed {processed_submissions} out of {download_info['items_found']} found")
self.logger.info(
f"[DEBUG] Final progress update - processed {processed_submissions} out of {download_info['items_found']} found"
)
progress_event = ProgressEvent(
"progress",
download_id,
@@ -686,26 +713,26 @@ class BDFRManager:
{
"phase": "completed",
"items_processed": processed_submissions,
"items_found": download_info["items_found"]
}
"items_found": download_info["items_found"],
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
elif hasattr(downloader, 'reddit_lists') and hasattr(downloader, 'write_entry'):
elif hasattr(downloader, "reddit_lists") and hasattr(downloader, "write_entry"):
# For Archiver, we need to intercept the archiving process
processed_items = 0
self.logger.info(f"[DEBUG] Archiver detected with reddit_lists and write_entry methods")
# Get the limit from config
limit = getattr(config, 'limit', 100)
limit = getattr(config, "limit", 100)
download_info["items_found"] = limit
self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}")
# Process items with progress tracking
self.logger.info(f"[DEBUG] Starting archiving loop...")
actual_count = 0
for generator in downloader.reddit_lists:
self.logger.info(f"[DEBUG] Processing generator for archiving...")
item_count = 0
@@ -714,14 +741,15 @@ class BDFRManager:
continue
item_count += 1
actual_count += 1
if download_info["status"] == DownloadStatus.CANCELLED.value:
self.logger.info(f"Download {download_id} was cancelled")
return
try:
# Determine item type
import praw.models
if isinstance(item, praw.models.Submission):
item_id = item.id
item_type = "submission"
@@ -729,23 +757,27 @@ class BDFRManager:
elif isinstance(item, praw.models.Comment):
item_id = item.id
item_type = "comment"
subreddit_name = item.subreddit.display_name if hasattr(item, 'subreddit') else "unknown"
subreddit_name = (
item.subreddit.display_name if hasattr(item, "subreddit") else "unknown"
)
else:
item_id = str(item)
item_type = "item"
subreddit_name = "unknown"
self.logger.info(f"[DEBUG] Archiving {item_type} {item_count}: {item_id} from r/{subreddit_name}")
self.logger.info(
f"[DEBUG] Archiving {item_type} {item_count}: {item_id} from r/{subreddit_name}"
)
# Update current item being processed
download_info["current_item"] = item_id
download_info["current_item_type"] = item_type
download_info["current_subreddit"] = subreddit_name
# Send progress update for this item
processed_items += 1
progress_percent = 20.0 + (processed_items / max(limit, 1)) * 70.0
progress_event = ProgressEvent(
"progress",
download_id,
@@ -757,11 +789,11 @@ class BDFRManager:
"current_item_type": item_type,
"current_subreddit": subreddit_name,
"items_processed": processed_items,
"items_found": download_info["items_found"]
}
"items_found": download_info["items_found"],
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
# Archive this item with retry mechanism for rate limiting
self.logger.info(f"[DEBUG] Calling write_entry for {item_id}")
try:
@@ -772,34 +804,37 @@ class BDFRManager:
error_msg = str(e)
if "429" in error_msg:
self.logger.warning(f"[DEBUG] Rate limited while archiving item {item_id}: {e}")
self.logger.warning(f"[DEBUG] Will continue with next item instead of failing entire download")
self.logger.warning(
f"[DEBUG] Will continue with next item instead of failing entire download"
)
continue
else:
# Not a rate limiting error, re-raise
raise
# Update processed count
download_info["items_processed"] = processed_items
except Exception as e:
import traceback
error_msg = str(e)
item_id = item_id if 'item_id' in locals() else "unknown"
item_id = item_id if "item_id" in locals() else "unknown"
stack_trace = traceback.format_exc()
if "429" in error_msg:
self.logger.error(f"[DEBUG] Rate limited while archiving item {item_id}: {e}")
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
download_info["status"] = DownloadStatus.FAILED.value
download_info["error"] = f"Rate limited by Reddit API: {e}"
download_info["end_time"] = datetime.now()
error_event = ProgressEvent(
"error",
download_id,
f"Rate limited by Reddit API: {e}",
None,
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"}
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"},
)
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
return
@@ -807,16 +842,18 @@ class BDFRManager:
self.logger.error(f"[DEBUG] Error archiving item {item_id}: {e}")
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
continue
self.logger.info(f"[DEBUG] Generator archived {item_count} items")
# Final progress update
if actual_count < limit:
download_info["items_found"] = actual_count
self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}")
self.logger.info(f"[DEBUG] Final progress update - archived {processed_items} out of {download_info['items_found']} found")
self.logger.info(
f"[DEBUG] Final progress update - archived {processed_items} out of {download_info['items_found']} found"
)
progress_event = ProgressEvent(
"progress",
download_id,
@@ -825,26 +862,32 @@ class BDFRManager:
{
"phase": "completed",
"items_processed": processed_items,
"items_found": download_info["items_found"]
}
"items_found": download_info["items_found"],
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
elif hasattr(downloader, 'reddit_lists') and hasattr(downloader, '_download_submission') and hasattr(downloader, 'write_entry'):
elif (
hasattr(downloader, "reddit_lists")
and hasattr(downloader, "_download_submission")
and hasattr(downloader, "write_entry")
):
# For RedditCloner (which has both download and archive capabilities)
processed_items = 0
self.logger.info(f"[DEBUG] RedditCloner detected with reddit_lists, _download_submission, and write_entry methods")
self.logger.info(
f"[DEBUG] RedditCloner detected with reddit_lists, _download_submission, and write_entry methods"
)
# Get the limit from config
limit = getattr(config, 'limit', 100)
limit = getattr(config, "limit", 100)
download_info["items_found"] = limit
self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}")
# Process submissions with progress tracking (both download and archive)
self.logger.info(f"[DEBUG] Starting cloning loop...")
actual_count = 0
for generator in downloader.reddit_lists:
self.logger.info(f"[DEBUG] Processing generator for cloning...")
submission_count = 0
@@ -853,23 +896,25 @@ class BDFRManager:
continue
submission_count += 1
actual_count += 1
if download_info["status"] == DownloadStatus.CANCELLED.value:
self.logger.info(f"Download {download_id} was cancelled")
return
try:
self.logger.info(f"[DEBUG] Cloning submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}")
self.logger.info(
f"[DEBUG] Cloning submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}"
)
# Update current item being processed
download_info["current_item"] = submission.id
download_info["current_item_type"] = "submission"
download_info["current_subreddit"] = submission.subreddit.display_name
# Send progress update for this submission
processed_items += 1
progress_percent = 20.0 + (processed_items / max(limit, 1)) * 70.0
progress_event = ProgressEvent(
"progress",
download_id,
@@ -881,11 +926,11 @@ class BDFRManager:
"current_item_type": "submission",
"current_subreddit": submission.subreddit.display_name,
"items_processed": processed_items,
"items_found": download_info["items_found"]
}
"items_found": download_info["items_found"],
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
# Clone this submission (download + archive) with retry mechanism for rate limiting
self.logger.info(f"[DEBUG] Calling _download_submission for {submission.id}")
try:
@@ -897,36 +942,43 @@ class BDFRManager:
# Check if this is a rate limiting error that should be retried
error_msg = str(e)
if "429" in error_msg:
self.logger.warning(f"[DEBUG] Rate limited while cloning submission {submission.id}: {e}")
self.logger.warning(f"[DEBUG] Will continue with next submission instead of failing entire download")
self.logger.warning(
f"[DEBUG] Rate limited while cloning submission {submission.id}: {e}"
)
self.logger.warning(
f"[DEBUG] Will continue with next submission instead of failing entire download"
)
continue
else:
# Not a rate limiting error, re-raise
raise
# Update processed count
download_info["items_processed"] = processed_items
except Exception as e:
import traceback
error_msg = str(e)
submission_id = submission.id if submission is not None else "unknown"
stack_trace = traceback.format_exc()
if "429" in error_msg:
self.logger.error(f"[DEBUG] Rate limited while cloning submission {submission_id}: {e}")
self.logger.error(
f"[DEBUG] Rate limited while cloning submission {submission_id}: {e}"
)
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
self.logger.info(f"[DEBUG] Error message contains '429': {error_msg}")
download_info["status"] = DownloadStatus.FAILED.value
download_info["error"] = f"Rate limited by Reddit API: {e}"
download_info["end_time"] = datetime.now()
error_event = ProgressEvent(
"error",
download_id,
f"Rate limited by Reddit API: {e}",
None,
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"}
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"},
)
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
return
@@ -934,16 +986,18 @@ class BDFRManager:
self.logger.error(f"[DEBUG] Error cloning submission {submission_id}: {e}")
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
continue
self.logger.info(f"[DEBUG] Generator cloned {submission_count} submissions")
# Final progress update
if actual_count < limit:
download_info["items_found"] = actual_count
self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}")
self.logger.info(f"[DEBUG] Final progress update - cloned {processed_items} out of {download_info['items_found']} found")
self.logger.info(
f"[DEBUG] Final progress update - cloned {processed_items} out of {download_info['items_found']} found"
)
progress_event = ProgressEvent(
"progress",
download_id,
@@ -952,11 +1006,11 @@ class BDFRManager:
{
"phase": "completed",
"items_processed": processed_items,
"items_found": download_info["items_found"]
}
"items_found": download_info["items_found"],
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
else:
# For other downloaders, just run normally
self.logger.info(f"[DEBUG] Using fallback downloader (no specific handler)")
@@ -964,6 +1018,7 @@ class BDFRManager:
except Exception as e:
import traceback
stack_trace = traceback.format_exc()
self.logger.error(f"Error in progress download for {download_id}: {e}")
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
@@ -971,7 +1026,7 @@ class BDFRManager:
raise
# Set up authentication if token provided
if hasattr(config, 'auth_token') and config.auth_token:
if hasattr(config, "auth_token") and config.auth_token:
self.logger.info(f"[DEBUG] Setting up authenticated Reddit instance")
# Monkey patch the create_reddit_instance method to use our token
original_create_reddit_instance = downloader.create_reddit_instance
@@ -985,11 +1040,12 @@ class BDFRManager:
# Override the reddit_instance with authenticated version
import praw
downloader.reddit_instance = praw.Reddit(
client_id=downloader.cfg_parser.get("DEFAULT", "client_id"),
client_secret=downloader.cfg_parser.get("DEFAULT", "client_secret"),
user_agent=getattr(downloader, 'user_agent', 'BDFR-Web-Interface/1.0'),
token=auth_token
user_agent=getattr(downloader, "user_agent", "BDFR-Web-Interface/1.0"),
token=auth_token,
)
downloader.authenticated = True
self.logger.info(f"[DEBUG] Created authenticated Reddit instance successfully")
@@ -1005,10 +1061,14 @@ class BDFRManager:
async def _notify_callbacks(self, callbacks: List[ProgressCallback], method_name: str, event: ProgressEvent):
"""Notify all callbacks of an event"""
self.logger.info(f"[BDFR-API] Notifying {len(callbacks)} callbacks of {method_name} event for download {event.download_id}")
self.logger.info(
f"[BDFR-API] Notifying {len(callbacks)} callbacks of {method_name} event for download {event.download_id}"
)
for callback in callbacks:
try:
logger.info(f"[BDFR-API] Invoking {method_name} on callback {callback.__class__.__name__} for download {event.download_id}")
logger.info(
f"[BDFR-API] Invoking {method_name} on callback {callback.__class__.__name__} for download {event.download_id}"
)
method = getattr(callback, method_name)
if asyncio.iscoroutinefunction(method):
await method(event)
@@ -1080,10 +1140,7 @@ class BDFRManager:
Returns:
Dictionary of download information keyed by download ID
"""
return {
download_id: self.get_download_status(download_id)
for download_id in self._active_downloads
}
return {download_id: self.get_download_status(download_id) for download_id in self._active_downloads}
def cleanup_completed(self, max_age_seconds: int = 3600) -> int:
"""
@@ -1099,7 +1156,11 @@ class BDFRManager:
to_remove = []
for download_id, download_info in self._active_downloads.items():
if download_info["status"] in [DownloadStatus.COMPLETED.value, DownloadStatus.FAILED.value, DownloadStatus.CANCELLED.value]:
if download_info["status"] in [
DownloadStatus.COMPLETED.value,
DownloadStatus.FAILED.value,
DownloadStatus.CANCELLED.value,
]:
end_time = download_info.get("end_time")
if end_time and (current_time - end_time).total_seconds() > max_age_seconds:
to_remove.append(download_id)
@@ -1124,7 +1185,7 @@ class BDFRManager:
time_filter: str = "all",
no_dupes: bool = False,
simple_check: bool = False,
progress_callbacks: Optional[List[ProgressCallback]] = None
progress_callbacks: Optional[List[ProgressCallback]] = None,
) -> str:
"""
Download content from a subreddit.
@@ -1168,12 +1229,7 @@ class BDFRManager:
config.no_dupes = no_dupes
config.simple_check = simple_check
download_id = self.create_download(
DownloadType.SUBREDDIT,
subreddit_name,
config,
progress_callbacks
)
download_id = self.create_download(DownloadType.SUBREDDIT, subreddit_name, config, progress_callbacks)
self.start_download(download_id)
return download_id
@@ -1187,7 +1243,7 @@ class BDFRManager:
saved: bool = False,
no_dupes: bool = False,
simple_check: bool = False,
progress_callbacks: Optional[List[ProgressCallback]] = None
progress_callbacks: Optional[List[ProgressCallback]] = None,
) -> str:
"""
Download content from a user.
@@ -1234,7 +1290,7 @@ class BDFRManager:
config.saved = saved
config.no_dupes = no_dupes
config.simple_check = simple_check
# Set authentication if token is available for this manager instance
if self.auth_token:
config.authenticate = True
@@ -1244,12 +1300,7 @@ class BDFRManager:
config.authenticate = False
logger.info(f"[DEBUG] No authentication token available for user download")
download_id = self.create_download(
DownloadType.USER,
username,
config,
progress_callbacks
)
download_id = self.create_download(DownloadType.USER, username, config, progress_callbacks)
self.start_download(download_id)
return download_id
@@ -1260,7 +1311,7 @@ class BDFRManager:
format_type: str = "json",
limit: Optional[int] = None,
simple_check: bool = False,
progress_callbacks: Optional[List[ProgressCallback]] = None
progress_callbacks: Optional[List[ProgressCallback]] = None,
) -> str:
"""
Archive subreddit data (metadata only, no downloads).
@@ -1298,12 +1349,7 @@ class BDFRManager:
config.limit = limit
config.format = format_type
download_id = self.create_download(
DownloadType.ARCHIVE,
subreddit_name,
config,
progress_callbacks
)
download_id = self.create_download(DownloadType.ARCHIVE, subreddit_name, config, progress_callbacks)
self.start_download(download_id)
return download_id
@@ -1315,7 +1361,7 @@ class BDFRManager:
format_type: str = "json",
no_dupes: bool = False,
simple_check: bool = False,
progress_callbacks: Optional[List[ProgressCallback]] = None
progress_callbacks: Optional[List[ProgressCallback]] = None,
) -> str:
"""
Clone subreddit (both download and archive).
@@ -1355,12 +1401,7 @@ class BDFRManager:
config.format = format_type
config.no_dupes = no_dupes
download_id = self.create_download(
DownloadType.CLONE,
subreddit_name,
config,
progress_callbacks
)
download_id = self.create_download(DownloadType.CLONE, subreddit_name, config, progress_callbacks)
self.start_download(download_id)
return download_id
@@ -1394,4 +1435,4 @@ def set_bdfr_manager(manager: BDFRManager):
manager: BDFRManager instance to use as default
"""
global _default_manager
_default_manager = manager
_default_manager = manager
bdfr/archiver.py
+3 -1
View File
@@ -52,7 +52,9 @@ class Archiver(RedditConnector):
logger.error(f"Submission {submission.id} failed to be archived due to a PRAW exception: {e}")
except prawcore.PrawcoreException as e:
if submission is not None:
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
logger.error(
f"The submission after {submission.id} failed to download due to a PRAW exception: {e}"
)
else:
logger.error(f"Download failed due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
bdfr/cloner.py
+3 -1
View File
@@ -30,7 +30,9 @@ class RedditCloner(RedditDownloader, Archiver):
logger.error(f"Submission {submission.id} failed to be cloned due to a PRAW exception: {e}")
except prawcore.PrawcoreException as e:
if submission is not None:
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
logger.error(
f"The submission after {submission.id} failed to download due to a PRAW exception: {e}"
)
else:
logger.error(f"Download failed due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
bdfr/connector.py
+11 -3
View File
@@ -414,9 +414,13 @@ class RedditConnector(metaclass=ABCMeta):
if TooManyRequests is not None and isinstance(e, TooManyRequests):
is_rate_limited = True
logger.info(f"Rate limited detected: Exception is TooManyRequests for user {user}")
elif (hasattr(e, "response") and getattr(e.response, "status_code", None) == 429) or "429" in str(e):
elif (hasattr(e, "response") and getattr(e.response, "status_code", None) == 429) or "429" in str(
e
):
is_rate_limited = True
logger.info(f"Rate limited detected: Status code 429 or '429' in error message for user {user}. Error: {e}")
logger.info(
f"Rate limited detected: Status code 429 or '429' in error message for user {user}. Error: {e}"
)
if is_rate_limited:
logger.error("Received HTTP 429 (rate limited). Propagating error to fail the download.")
@@ -441,7 +445,11 @@ class RedditConnector(metaclass=ABCMeta):
def create_file_name_formatter(self) -> FileNameFormatter:
return FileNameFormatter(
self.args.file_scheme, self.args.folder_scheme, self.args.time_format, self.args.filename_restriction_scheme, self.args.strip_unicode
self.args.file_scheme,
self.args.folder_scheme,
self.args.time_format,
self.args.filename_restriction_scheme,
self.args.strip_unicode,
)
def create_time_filter(self) -> RedditTypes.TimeType:
bdfr/downloader.py
+39 -39
View File
@@ -50,11 +50,13 @@ class RedditDownloader(RedditConnector):
hash_data = self._load_hash_list()
# Handle both old and new hash file formats
if isinstance(hash_data, dict) and 'files' in hash_data:
if isinstance(hash_data, dict) and "files" in hash_data:
# New format with enhanced structure
self.master_hash_list = {k: v['path'] for k, v in hash_data['files'].items()}
self.url_list = hash_data.get('urls', {})
logger.info(f"Loaded {len(self.master_hash_list)} hashes and {len(self.url_list)} URLs from enhanced hash file")
self.master_hash_list = {k: v["path"] for k, v in hash_data["files"].items()}
self.url_list = hash_data.get("urls", {})
logger.info(
f"Loaded {len(self.master_hash_list)} hashes and {len(self.url_list)} URLs from enhanced hash file"
)
else:
# Old format - just hashes
self.master_hash_list = hash_data
@@ -70,8 +72,10 @@ class RedditDownloader(RedditConnector):
if hash_value not in existing_hashes:
self.master_hash_list[hash_value] = file_path
logger.info(f"Loaded {len(self.master_hash_list)} total hashes "
f"({len(existing_hashes)} from file, {len(all_files_hashes) - len(existing_hashes)} new)")
logger.info(
f"Loaded {len(self.master_hash_list)} total hashes "
f"({len(existing_hashes)} from file, {len(all_files_hashes) - len(existing_hashes)} new)"
)
def download(self):
for generator in self.reddit_lists:
@@ -164,7 +168,7 @@ class RedditDownloader(RedditConnector):
self.master_hash_list[existing_file_hash] = destination
# Store URL mapping for simple-check functionality if URL is available
if hasattr(res, 'url') and self.args.simple_check:
if hasattr(res, "url") and self.args.simple_check:
self.url_list[res.url] = existing_file_hash
logger.debug(f"Added hash for existing file: {existing_file_hash}")
@@ -190,7 +194,7 @@ class RedditDownloader(RedditConnector):
resource_hash = res.hash.hexdigest()
# Simple-check: URL-based duplicate detection (fast path)
if self.args.simple_check and hasattr(res, 'url') and res.url in self.url_list:
if self.args.simple_check and hasattr(res, "url") and res.url in self.url_list:
stored_hash = self.url_list[res.url]
if stored_hash in self.master_hash_list:
logger.info(f"URL {res.url} from submission {submission.id} already downloaded (simple-check)")
@@ -233,7 +237,7 @@ class RedditDownloader(RedditConnector):
self.master_hash_list[resource_hash] = destination
# Store URL mapping for simple-check functionality
if hasattr(res, 'url') and self.args.simple_check:
if hasattr(res, "url") and self.args.simple_check:
self.url_list[res.url] = resource_hash
logger.debug(f"Hash added to master list: {resource_hash}")
@@ -252,7 +256,7 @@ class RedditDownloader(RedditConnector):
@staticmethod
def scan_existing_files(directory: Path) -> dict[str, Path]:
files = []
for (dirpath, _dirnames, filenames) in os.walk(directory):
for dirpath, _dirnames, filenames in os.walk(directory):
files.extend([Path(dirpath, file) for file in filenames])
logger.info(f"Calculating hashes for {len(files)} files")
@@ -270,14 +274,14 @@ class RedditDownloader(RedditConnector):
def _load_hash_list(self) -> dict[str, Path]:
"""Load existing hash list from .bdfr_hashes.json in download directory."""
logger.debug(f"Loading hash list from directory: {self.download_directory}")
hash_file_path = self.download_directory / '.bdfr_hashes.json'
hash_file_path = self.download_directory / ".bdfr_hashes.json"
if not hash_file_path.exists():
logger.debug(f"No existing hash file found at {hash_file_path}")
return {}
try:
with open(hash_file_path, 'r', encoding='utf-8') as f:
with open(hash_file_path, "r", encoding="utf-8") as f:
hash_data = json.load(f)
if not isinstance(hash_data, dict):
@@ -285,16 +289,16 @@ class RedditDownloader(RedditConnector):
return {}
# Handle new enhanced format
if 'files' in hash_data and isinstance(hash_data['files'], dict):
if "files" in hash_data and isinstance(hash_data["files"], dict):
# New format with enhanced structure
files_data = hash_data['files']
files_data = hash_data["files"]
loaded_hashes = {}
urls_data = hash_data.get('urls', {})
urls_data = hash_data.get("urls", {})
for hash_value, file_info in files_data.items():
if isinstance(file_info, dict) and 'path' in file_info:
if isinstance(file_info, dict) and "path" in file_info:
# New format: {"hash": {"path": "relative/path", "url": "http://..."}}
relative_path = file_info['path']
relative_path = file_info["path"]
absolute_path = self.download_directory / relative_path
if absolute_path.exists():
loaded_hashes[hash_value] = absolute_path
@@ -302,8 +306,8 @@ class RedditDownloader(RedditConnector):
logger.debug(f"File {absolute_path} from hash file no longer exists")
# Load URL mapping for simple-check
if 'url' in file_info and file_info['url']:
self.url_list[file_info['url']] = hash_value
if "url" in file_info and file_info["url"]:
self.url_list[file_info["url"]] = hash_value
elif isinstance(file_info, str):
# Legacy format within new structure: {"hash": "relative/path"}
absolute_path = self.download_directory / file_info
@@ -337,30 +341,30 @@ class RedditDownloader(RedditConnector):
def _save_hash_list(self) -> None:
"""Save current hash list to .bdfr_hashes.json in download directory using atomic write."""
hash_file_path = self.download_directory / '.bdfr_hashes.json'
hash_file_path = self.download_directory / ".bdfr_hashes.json"
# Build enhanced data structure for new format
if self.args.simple_check:
# New enhanced format with URLs and metadata
hash_data = {
'files': {},
'urls': self.url_list.copy(),
'metadata': {
'version': '2.0',
'created_with': 'simple_check' if self.args.simple_check else 'standard',
'url_count': len(self.url_list),
'hash_count': len(self.master_hash_list)
}
"files": {},
"urls": self.url_list.copy(),
"metadata": {
"version": "2.0",
"created_with": "simple_check" if self.args.simple_check else "standard",
"url_count": len(self.url_list),
"hash_count": len(self.master_hash_list),
},
}
# Convert absolute paths to relative paths for portability
for hash_value, absolute_path in self.master_hash_list.items():
try:
relative_path = absolute_path.relative_to(self.download_directory)
hash_data['files'][hash_value] = {
'path': str(relative_path),
'url': next((url for url, h in self.url_list.items() if h == hash_value), None),
'check_method': 'hash'
hash_data["files"][hash_value] = {
"path": str(relative_path),
"url": next((url for url, h in self.url_list.items() if h == hash_value), None),
"check_method": "hash",
}
except ValueError:
# File is not relative to download directory, skip it
@@ -381,17 +385,13 @@ class RedditDownloader(RedditConnector):
# Atomic write: write to temporary file first, then rename
try:
with tempfile.NamedTemporaryFile(
mode='w',
dir=self.download_directory,
suffix='.tmp',
delete=False,
encoding='utf-8'
mode="w", dir=self.download_directory, suffix=".tmp", delete=False, encoding="utf-8"
) as temp_file:
json.dump(hash_data, temp_file, indent=2)
temp_file_path = temp_file.name
# Atomic rename
if os.name == 'nt': # Windows
if os.name == "nt": # Windows
# On Windows, we need to remove the target file first if it exists
if hash_file_path.exists():
hash_file_path.unlink()
@@ -407,7 +407,7 @@ class RedditDownloader(RedditConnector):
logger.error(f"Unexpected error saving hash file {hash_file_path}: {e}")
# Clean up temp file if it still exists
try:
if 'temp_file_path' in locals():
if "temp_file_path" in locals():
os.unlink(temp_file_path)
except (OSError, IOError):
pass
bdfr/examples/api_usage.py
+24 -50
View File
@@ -19,20 +19,16 @@ sys.path.insert(0, str(Path(__file__).parent.parent))
from bdfr.api import (
BDFRManager,
DownloadType,
DownloadStatus,
ProgressEvent,
ProgressCallback,
DownloadType,
LoggingCallback,
get_bdfr_manager
ProgressCallback,
ProgressEvent,
get_bdfr_manager,
)
# Configure logging
logging.basicConfig(
level=logging.INFO,
format='[%(asctime)s] %(levelname)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S'
)
logging.basicConfig(level=logging.INFO, format="[%(asctime)s] %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
logger = logging.getLogger(__name__)
@@ -53,7 +49,7 @@ class WebSocketCallback(ProgressCallback):
async def on_error(self, event: ProgressEvent):
"""Send error update to WebSocket"""
print(f"❌ [{self.websocket_id}] ERROR: {event.message}")
if event.data.get('exception'):
if event.data.get("exception"):
print(f" Exception: {event.data['exception']}")
async def on_completed(self, event: ProgressEvent):
@@ -95,9 +91,9 @@ async def example_basic_usage():
# Create a download for a subreddit
download_id = manager.download_subreddit(
"python", # subreddit name
limit=10, # download 10 posts
sort="hot", # sort by hot
no_dupes=True # avoid duplicates
limit=10, # download 10 posts
sort="hot", # sort by hot
no_dupes=True, # avoid duplicates
)
print(f"📋 Created download with ID: {download_id}")
@@ -111,7 +107,7 @@ async def example_basic_usage():
print(f"📊 Status: {status['status']} | Progress: {int(round(status['progress']))}%")
if status['status'] in ['completed', 'failed', 'cancelled']:
if status["status"] in ["completed", "failed", "cancelled"]:
print(f"🏁 Download finished with status: {status['status']}")
break
@@ -126,11 +122,7 @@ async def example_advanced_usage():
print("=" * 50)
# Create custom callbacks
callbacks = [
LoggingCallback("web_interface"),
WebSocketCallback("user_123"),
DatabaseCallback()
]
callbacks = [LoggingCallback("web_interface"), WebSocketCallback("user_123"), DatabaseCallback()]
# Create manager with custom download directory
manager = BDFRManager("./custom_downloads")
@@ -140,11 +132,7 @@ async def example_advanced_usage():
download_ids = []
for subreddit in subreddits:
download_id = manager.create_download(
DownloadType.SUBREDDIT,
subreddit,
progress_callbacks=callbacks
)
download_id = manager.create_download(DownloadType.SUBREDDIT, subreddit, progress_callbacks=callbacks)
# Start the download
manager.start_download(download_id)
@@ -164,7 +152,7 @@ async def example_advanced_usage():
print(f"📊 {download_id}: {status['status']} ({int(round(status['progress']))}%)")
if status['status'] in ['completed', 'failed', 'cancelled']:
if status["status"] in ["completed", "failed", "cancelled"]:
print(f"🏁 Download {download_id} finished")
download_ids.remove(download_id)
else:
@@ -187,11 +175,7 @@ async def example_user_download():
# Download user's submitted posts
download_id = manager.download_user(
"testuser", # username
limit=25, # 25 posts
submitted=True,
upvoted=False,
saved=False
"testuser", limit=25, submitted=True, upvoted=False, saved=False # username # 25 posts
)
print(f"📋 Created user download: {download_id}")
@@ -205,7 +189,7 @@ async def example_user_download():
print(f"📊 Status: {status['status']} | Progress: {int(round(status['progress']))}%")
if status['status'] in ['completed', 'failed']:
if status["status"] in ["completed", "failed"]:
break
await asyncio.sleep(2)
@@ -221,11 +205,7 @@ async def example_archive_operation():
manager = BDFRManager("./archives")
# Archive subreddit data (metadata only)
download_id = manager.archive_subreddit(
"dataisbeautiful",
format_type="json",
limit=50
)
download_id = manager.archive_subreddit("dataisbeautiful", format_type="json", limit=50)
print(f"📋 Created archive operation: {download_id}")
@@ -238,7 +218,7 @@ async def example_archive_operation():
print(f"📊 Archive status: {status['status']} | Progress: {int(round(status['progress']))}%")
if status['status'] in ['completed', 'failed']:
if status["status"] in ["completed", "failed"]:
print(f"🏁 Archive finished with status: {status['status']}")
break
@@ -265,11 +245,7 @@ async def example_web_integration():
callback = WebSocketCallback(f"ws_{user_id}")
# Create and start download
download_id = self.manager.download_subreddit(
subreddit,
limit=limit,
progress_callbacks=[callback]
)
download_id = self.manager.download_subreddit(subreddit, limit=limit, progress_callbacks=[callback])
# Track for this user session
if user_id not in self.active_sessions:
@@ -279,7 +255,7 @@ async def example_web_integration():
return {
"success": True,
"download_id": download_id,
"message": f"Started download of r/{subreddit} (limit: {limit})"
"message": f"Started download of r/{subreddit} (limit: {limit})",
}
async def get_user_downloads(self, user_id: str):
@@ -329,7 +305,7 @@ async def example_web_integration():
# Cancel one download
if user_downloads:
cancel_result = await app.cancel_user_download(user_id, user_downloads[0]['id'])
cancel_result = await app.cancel_user_download(user_id, user_downloads[0]["id"])
print(f"Cancel result: {cancel_result}")
return len(user_downloads)
@@ -343,10 +319,7 @@ async def example_error_handling():
manager = BDFRManager("./test_downloads")
# Try to download from a non-existent subreddit
download_id = manager.download_subreddit(
"this_subreddit_does_not_exist",
limit=5
)
download_id = manager.download_subreddit("this_subreddit_does_not_exist", limit=5)
print(f"📋 Created download for non-existent subreddit: {download_id}")
@@ -359,7 +332,7 @@ async def example_error_handling():
print(f"📊 Status: {status['status']}")
if status['status'] == 'failed':
if status["status"] == "failed":
print(f"🏁 Download failed as expected: {status.get('error', 'Unknown error')}")
break
@@ -401,9 +374,10 @@ async def main():
except Exception as e:
print(f"\n❌ Error running examples: {e}")
import traceback
traceback.print_exc()
if __name__ == "__main__":
# Run the examples
asyncio.run(main())
asyncio.run(main())
bdfr/file_name_formatter.py
+19 -3
View File
@@ -245,10 +245,26 @@ class FileNameFormatter:
if ord(char) < 0x80:
result.append(char)
# Keep common Unicode letters, numbers, and punctuation
elif unicodedata.category(char) in ['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Ps', 'Pe', 'Pi', 'Pf', 'Po']:
elif unicodedata.category(char) in [
"Lu",
"Ll",
"Lt",
"Lm",
"Lo",
"Nd",
"Nl",
"No",
"Pc",
"Pd",
"Ps",
"Pe",
"Pi",
"Pf",
"Po",
]:
result.append(char)
# Strip emoji, symbols, and other special characters that cause 8.3 names
elif unicodedata.category(char).startswith(('S', 'So', 'Sk', 'Sm')): # Symbols
elif unicodedata.category(char).startswith(("S", "So", "Sk", "Sm")): # Symbols
continue
elif ord(char) > 0x1F000: # High Unicode ranges often contain emoji
continue
@@ -256,7 +272,7 @@ class FileNameFormatter:
# Keep other Unicode characters that are generally safe
result.append(char)
return ''.join(result)
return "".join(result)
@staticmethod
def _strip_emojis(input_string: str) -> str:
bdfr/resource.py
+22 -20
View File
@@ -24,11 +24,11 @@ class Resource:
self.content: Optional[bytes] = None
self.url = url
self.hash: Optional[_hashlib.HASH] = None
# Log the original extension before normalization
if extension:
logger.debug(f"Resource constructor received extension: '{extension}' for URL: {url}")
self.extension = self._normalize_extension(extension)
self.download_function = download_function
if not self.extension:
@@ -72,14 +72,14 @@ class Resource:
if self.url.startswith("https://www.reddit.com/media"):
logger.debug(f"Detected Reddit media URL: {self.url}")
parsed_url = urllib.parse.urlparse(self.url)
url_param = urllib.parse.parse_qs(parsed_url.query).get('url', [None])[0]
url_param = urllib.parse.parse_qs(parsed_url.query).get("url", [None])[0]
if url_param:
decoded_url = urllib.parse.unquote(url_param)
logger.debug(f"Reddit media URL decoded to: {decoded_url}")
stripped_url = urllib.parse.urlsplit(decoded_url).path
# Also handle preview.redd.it URLs which might not have extensions
elif "preview.redd.it" in self.url and not stripped_url.endswith(('.jpg', '.jpeg', '.png', '.gif', '.webp')):
elif "preview.redd.it" in self.url and not stripped_url.endswith((".jpg", ".jpeg", ".png", ".gif", ".webp")):
logger.debug(f"Detected preview.redd.it URL without extension: {self.url}")
# For preview URLs, try to infer from common patterns or add fallback logic
@@ -92,7 +92,7 @@ class Resource:
logger.warning(f"Could not determine extension for URL: {self.url} (path: {stripped_url})")
# As a last resort, if we have content, try to detect by magic numbers
if hasattr(self, 'content') and self.content:
if hasattr(self, "content") and self.content:
detected = self._detect_extension_by_content()
return self._normalize_extension(detected) if detected else None
@@ -104,21 +104,21 @@ class Resource:
return None
# Check for common image formats
if self.content.startswith(b'\xFF\xD8\xFF'):
if self.content.startswith(b"\xff\xd8\xff"):
logger.debug(f"Detected JPEG by magic number for URL: {self.url}")
return '.jpg'
elif self.content.startswith(b'\x89PNG\r\n\x1a\n'):
return ".jpg"
elif self.content.startswith(b"\x89PNG\r\n\x1a\n"):
logger.debug(f"Detected PNG by magic number for URL: {self.url}")
return '.png'
elif self.content.startswith(b'GIF87a') or self.content.startswith(b'GIF89a'):
return ".png"
elif self.content.startswith(b"GIF87a") or self.content.startswith(b"GIF89a"):
logger.debug(f"Detected GIF by magic number for URL: {self.url}")
return '.gif'
elif self.content.startswith(b'RIFF') and self.content[8:12] == b'WEBP':
return ".gif"
elif self.content.startswith(b"RIFF") and self.content[8:12] == b"WEBP":
logger.debug(f"Detected WebP by magic number for URL: {self.url}")
return '.webp'
elif self.content.startswith(b'BM'):
return ".webp"
elif self.content.startswith(b"BM"):
logger.debug(f"Detected BMP by magic number for URL: {self.url}")
return '.bmp'
return ".bmp"
logger.debug(f"Could not detect file type by magic number for URL: {self.url}")
return None
@@ -127,15 +127,17 @@ class Resource:
"""Normalize extension to lowercase for consistency"""
if not extension:
return None
original = extension
# Ensure extension starts with a dot
if not extension.startswith('.'):
extension = '.' + extension
if not extension.startswith("."):
extension = "." + extension
normalized = extension.lower()
if original != normalized:
logger.info(f"Extension normalization: '{original}' -> '{normalized}' for URL: {self.url if hasattr(self, 'url') else 'unknown'}")
logger.info(
f"Extension normalization: '{original}' -> '{normalized}' for URL: {self.url if hasattr(self, 'url') else 'unknown'}"
)
return normalized
@staticmethod
bdfr/site_downloaders/download_factory.py
+1
View File
@@ -25,6 +25,7 @@ class DownloadFactory:
@staticmethod
def pull_lever(url: str) -> type[BaseDownloader]:
import logging
logger = logging.getLogger(__name__)
sanitised_url = DownloadFactory.sanitise_url(url).lower()