ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: ModerateWinGuy/BDFR_Web:f3dc8e46fdc093a2d9bbe71e2905672a8d4a5f95
Branches Tags
ModerateWinGuy/BDFR_Web:master
...
compare: ModerateWinGuy/BDFR_Web:master
Branches Tags
ModerateWinGuy/BDFR_Web:master
2 Commits
f3dc8e46fd ... master
Author SHA1 Message Date
ModerateWinGuy 0157f462cc reformatting
formatting_check / formatting_check (push) Failing after 3s
Details
Python Test / test (.ps1, windows-latest, 3.9) (push) Has been cancelled
Details
Python Test / test (.sh, macos-latest, 3.9) (push) Has been cancelled
Details
Python Test / test (.sh, ubuntu-latest, 3.9) (push) Has been cancelled
Details
2026-07-14 21:32:55 +12:00
ModerateWinGuy 3d0658b483 fixed lintng
formatting_check / formatting_check (push) Failing after 4s
Details
Python Test / test (.ps1, windows-latest, 3.9) (push) Has been cancelled
Details
Python Test / test (.sh, macos-latest, 3.9) (push) Has been cancelled
Details
Python Test / test (.sh, ubuntu-latest, 3.9) (push) Has been cancelled
Details
2026-07-14 21:23:30 +12:00
18 changed files with 439 additions and 378 deletions
Expand all files Collapse all files
bdfr/__main__.py
+11 -2
View File
@@ -26,7 +26,11 @@ _common_options = [
click.option("--file-scheme", default=None, type=str),
click.option("--filename-restriction-scheme", type=click.Choice(("linux", "windows")), default=None),
click.option("--folder-scheme", default=None, type=str),
click.option("--strip-unicode/--no-strip-unicode", default=None, help="Strip Unicode characters that cause Windows SMB issues (default: enabled)"),
click.option(
"--strip-unicode/--no-strip-unicode",
default=None,
help="Strip Unicode characters that cause Windows SMB issues (default: enabled)",
),
click.option("--ignore-user", type=str, multiple=True, default=None),
click.option("--include-id-file", multiple=True, default=None),
click.option("--log", type=str, default=None),
@@ -54,7 +58,12 @@ _downloader_options = [
click.option("--max-wait-time", type=int, default=None),
click.option("--no-dupes", is_flag=True, default=None),
click.option("--search-existing", is_flag=True, default=None),
click.option("--simple-check", is_flag=True, default=None, help="Enable fast URL-based duplicate checking (works with --no-dupes)"),
click.option(
"--simple-check",
is_flag=True,
default=None,
help="Enable fast URL-based duplicate checking (works with --no-dupes)",
),
click.option("--skip", default=None, multiple=True),
click.option("--skip-domain", default=None, multiple=True),
click.option("--skip-subreddit", default=None, multiple=True),
bdfr/api.py
+194 -153
View File
@@ -25,12 +25,12 @@ from typing import Any, Dict, List, Optional, Union
import prawcore
from bdfr import exceptions as errors
from bdfr.archiver import Archiver
from bdfr.cloner import RedditCloner
from bdfr.configuration import Configuration
from bdfr.connector import RedditConnector
from bdfr.downloader import RedditDownloader
from bdfr.archiver import Archiver
from bdfr.cloner import RedditCloner
from bdfr import exceptions as errors
logger = logging.getLogger(__name__)
@@ -84,7 +84,9 @@ def retry_reddit_api_call(api_call_func: Callable, max_retries: int = 5, base_wa
raise Exception(f"Reddit API rate limited after {max_retries + 1} attempts: {e}")
# Wait before retrying
logger.info(f"Reddit API rate limited, waiting {current_wait_time} seconds before retry {attempt + 1}/{max_retries}")
logger.info(
f"Reddit API rate limited, waiting {current_wait_time} seconds before retry {attempt + 1}/{max_retries}"
)
time.sleep(current_wait_time)
# Increase wait time for next attempt (exponential backoff)
@@ -93,6 +95,7 @@ def retry_reddit_api_call(api_call_func: Callable, max_retries: int = 5, base_wa
class DownloadType(Enum):
"""Types of downloads supported by BDFR"""
SUBREDDIT = "subreddit"
USER = "user"
MULTIREDDIT = "multireddit"
@@ -103,6 +106,7 @@ class DownloadType(Enum):
class DownloadStatus(Enum):
"""Status of a download operation"""
QUEUED = "queued"
RUNNING = "running"
COMPLETED = "completed"
@@ -121,7 +125,7 @@ class ProgressEvent:
message: str,
progress: Optional[float] = None,
data: Optional[Dict[str, Any]] = None,
timestamp: Optional[datetime] = None
timestamp: Optional[datetime] = None,
):
self.event_type = event_type # "progress", "status", "error", "completed"
self.download_id = download_id
@@ -138,7 +142,7 @@ class ProgressEvent:
"message": self.message,
"progress": self.progress,
"data": self.data,
"timestamp": self.timestamp.isoformat()
"timestamp": self.timestamp.isoformat(),
}
@@ -177,8 +181,8 @@ class LoggingCallback(ProgressCallback):
async def on_error(self, event: ProgressEvent):
"""Log error events"""
self.logger.error(f"[{event.download_id}] ERROR: {event.message}")
if event.data.get('exception'):
self.logger.exception(f"[{event.download_id}] Exception details", exc_info=event.data['exception'])
if event.data.get("exception"):
self.logger.exception(f"[{event.download_id}] Exception details", exc_info=event.data["exception"])
async def on_completed(self, event: ProgressEvent):
"""Log completion events"""
@@ -258,7 +262,7 @@ class BDFRManager:
download_type: DownloadType,
name: str,
config: Optional[Configuration] = None,
progress_callbacks: Optional[List[ProgressCallback]] = None
progress_callbacks: Optional[List[ProgressCallback]] = None,
) -> str:
"""
Create a new download operation.
@@ -326,7 +330,7 @@ class BDFRManager:
"items_processed": 0,
"items_found": 0,
"current_item": None,
"phase": "queued"
"phase": "queued",
}
self.logger.info(f"Created download {download_id} for {download_type.value}: {name}")
@@ -356,11 +360,7 @@ class BDFRManager:
download_info["start_time"] = datetime.now()
# Create and start download thread
download_thread = threading.Thread(
target=self._run_download,
args=(download_id,),
daemon=True
)
download_thread = threading.Thread(target=self._run_download, args=(download_id,), daemon=True)
self._download_threads[download_id] = download_thread
download_thread.start()
@@ -375,8 +375,12 @@ class BDFRManager:
downloader = None
try:
self.logger.info(f"[DEBUG] Starting download {download_id} for {download_info['type']}: {download_info['name']}")
self.logger.info(f"[DEBUG] Config: subreddit={getattr(config, 'subreddit', None)}, user={getattr(config, 'user', None)}, limit={getattr(config, 'limit', None)}")
self.logger.info(
f"[DEBUG] Starting download {download_id} for {download_info['type']}: {download_info['name']}"
)
self.logger.info(
f"[DEBUG] Config: subreddit={getattr(config, 'subreddit', None)}, user={getattr(config, 'user', None)}, limit={getattr(config, 'limit', None)}"
)
# Create progress event for start
start_event = ProgressEvent(
@@ -384,11 +388,13 @@ class BDFRManager:
download_id,
f"Starting {download_info['type']} download: {download_info['name']}",
0.0,
{"phase": "starting"}
{"phase": "starting"},
)
# Notify callbacks
self.logger.info(f"[BDFR-API] Notifying {len(callbacks)} callbacks of progress event for download {download_id}")
self.logger.info(
f"[BDFR-API] Notifying {len(callbacks)} callbacks of progress event for download {download_id}"
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", start_event))
# Set up authentication token in config if provided
@@ -411,7 +417,9 @@ class BDFRManager:
log_path.parent.mkdir(parents=True, exist_ok=True)
# Determine which BDFR class to use based on configuration
self.logger.info(f"[DEBUG] Determining downloader type - submitted={getattr(config, 'submitted', False)}, upvoted={getattr(config, 'upvoted', False)}, saved={getattr(config, 'saved', False)}, format={getattr(config, 'format', 'N/A')}")
self.logger.info(
f"[DEBUG] Determining downloader type - submitted={getattr(config, 'submitted', False)}, upvoted={getattr(config, 'upvoted', False)}, saved={getattr(config, 'saved', False)}, format={getattr(config, 'format', 'N/A')}"
)
try:
if config.submitted or config.upvoted or config.saved:
@@ -420,7 +428,7 @@ class BDFRManager:
downloader = RedditDownloader(config, [])
download_info["operation_type"] = "user_download"
self.logger.info(f"[DEBUG] Using RedditDownloader for user data")
elif hasattr(config, 'format') and config.format in ["json", "xml", "yaml"]:
elif hasattr(config, "format") and config.format in ["json", "xml", "yaml"]:
# Archive operation
self.logger.info(f"[DEBUG] Creating Archiver for archive operation")
downloader = Archiver(config, [])
@@ -447,7 +455,9 @@ class BDFRManager:
# Mark as completed only if not already failed or cancelled
if download_info.get("status") not in [DownloadStatus.FAILED.value, DownloadStatus.CANCELLED.value]:
self.logger.info(f"[DEBUG] Marking download {download_id} as completed")
self.logger.info(f"[DEBUG] Final stats - items_processed: {download_info['items_processed']}, items_found: {download_info['items_found']}")
self.logger.info(
f"[DEBUG] Final stats - items_processed: {download_info['items_processed']}, items_found: {download_info['items_found']}"
)
download_info["status"] = DownloadStatus.COMPLETED.value
download_info["progress"] = 100.0
download_info["end_time"] = datetime.now()
@@ -460,18 +470,21 @@ class BDFRManager:
{
"items_processed": download_info["items_processed"],
"items_found": download_info["items_found"],
"operation_type": download_info.get("operation_type")
}
"operation_type": download_info.get("operation_type"),
},
)
self.logger.info(f"[DEBUG] Sending completion callback for {download_id}")
self.logger.info(f"[BDFR-API] Sending completion event for download {download_id}")
asyncio.run(self._notify_callbacks(callbacks, "on_completed", completed_event))
else:
self.logger.info(f"[DEBUG] Skipping completion for {download_id} due to status={download_info.get('status')}")
self.logger.info(
f"[DEBUG] Skipping completion for {download_id} due to status={download_info.get('status')}"
)
except Exception as e:
import traceback
# Get full stack trace for debugging
stack_trace = traceback.format_exc()
@@ -479,6 +492,7 @@ class BDFRManager:
is_rate_limited = False
try:
import prawcore
TooManyRequests = getattr(prawcore.exceptions, "TooManyRequests", None)
if TooManyRequests is not None and isinstance(e, TooManyRequests):
is_rate_limited = True
@@ -503,7 +517,11 @@ class BDFRManager:
download_id,
f"Download failed: {str(e)}",
None,
{"exception": str(e), "stack_trace": stack_trace, "phase": ("rate_limited" if is_rate_limited else "failed")}
{
"exception": str(e),
"stack_trace": stack_trace,
"phase": ("rate_limited" if is_rate_limited else "failed"),
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
@@ -519,17 +537,19 @@ class BDFRManager:
# Close any logging handlers that might be holding file locks
logger = logging.getLogger()
for handler in logger.handlers[:]:
if hasattr(handler, 'close') and hasattr(handler, 'baseFilename'):
if hasattr(handler, "close") and hasattr(handler, "baseFilename"):
# Check if this handler is for our download's log file
if hasattr(config, 'log') and config.log:
handler_path = getattr(handler, 'baseFilename', '')
if hasattr(config, "log") and config.log:
handler_path = getattr(handler, "baseFilename", "")
if handler_path == config.log or handler_path.startswith(str(self.download_directory)):
try:
handler.close()
logger.removeHandler(handler)
self.logger.info(f"[DEBUG] Closed logging handler for {handler_path}")
except Exception as cleanup_error:
self.logger.warning(f"[DEBUG] Error closing handler {handler_path}: {cleanup_error}")
self.logger.warning(
f"[DEBUG] Error closing handler {handler_path}: {cleanup_error}"
)
except Exception as cleanup_error:
self.logger.warning(f"[DEBUG] Error during logging cleanup: {cleanup_error}")
@@ -549,17 +569,13 @@ class BDFRManager:
# Update phase to fetching
download_info["phase"] = "fetching_submissions"
progress_event = ProgressEvent(
"progress",
download_id,
"Fetching submissions...",
10.0,
{"phase": "fetching_submissions"}
"progress", download_id, "Fetching submissions...", 10.0, {"phase": "fetching_submissions"}
)
self.logger.info(f"[BDFR-API] Sending progress update for download {download_id}")
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
# For RedditDownloader, we need to intercept the download process
if hasattr(downloader, 'reddit_lists') and hasattr(downloader, '_download_submission'):
if hasattr(downloader, "reddit_lists") and hasattr(downloader, "_download_submission"):
processed_submissions = 0
self.logger.info(f"[DEBUG] RedditDownloader has reddit_lists and _download_submission methods")
@@ -567,7 +583,7 @@ class BDFRManager:
# Get the limit from config - this is our target/goal for items_found
# items_found represents "how many items are available to download (up to limit)"
# items_processed represents "how many we've actually processed so far"
limit = getattr(config, 'limit', 100)
limit = getattr(config, "limit", 100)
download_info["items_found"] = limit
self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}")
@@ -590,7 +606,9 @@ class BDFRManager:
return
try:
self.logger.info(f"[DEBUG] Processing submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}")
self.logger.info(
f"[DEBUG] Processing submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}"
)
# Update current item being processed
download_info["current_item"] = submission.id
@@ -614,8 +632,8 @@ class BDFRManager:
"current_item_type": "submission",
"current_subreddit": submission.subreddit.display_name,
"items_processed": processed_submissions,
"items_found": download_info["items_found"] # Keep showing the limit as target
}
"items_found": download_info["items_found"], # Keep showing the limit as target
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
@@ -628,8 +646,12 @@ class BDFRManager:
# Check if this is a rate limiting error that should be retried
error_msg = str(e)
if "429" in error_msg:
self.logger.warning(f"[DEBUG] Rate limited while processing submission {submission.id}: {e}")
self.logger.warning(f"[DEBUG] Will continue with next submission instead of failing entire download")
self.logger.warning(
f"[DEBUG] Rate limited while processing submission {submission.id}: {e}"
)
self.logger.warning(
f"[DEBUG] Will continue with next submission instead of failing entire download"
)
continue
else:
# Not a rate limiting error, re-raise
@@ -640,6 +662,7 @@ class BDFRManager:
except Exception as e:
import traceback
error_msg = str(e)
submission_id = submission.id if submission is not None else "unknown"
@@ -647,7 +670,9 @@ class BDFRManager:
stack_trace = traceback.format_exc()
if "429" in error_msg:
self.logger.error(f"[DEBUG] Rate limited while processing submission {submission_id}: {e}")
self.logger.error(
f"[DEBUG] Rate limited while processing submission {submission_id}: {e}"
)
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
# Mark download as failed due to rate limiting
download_info["status"] = DownloadStatus.FAILED.value
@@ -659,7 +684,7 @@ class BDFRManager:
download_id,
f"Rate limited by Reddit API: {e}",
None,
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"}
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"},
)
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
return
@@ -675,9 +700,11 @@ class BDFRManager:
if actual_count < limit:
download_info["items_found"] = actual_count
self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}")
self.logger.info(f"[DEBUG] Final progress update - processed {processed_submissions} out of {download_info['items_found']} found")
self.logger.info(
f"[DEBUG] Final progress update - processed {processed_submissions} out of {download_info['items_found']} found"
)
progress_event = ProgressEvent(
"progress",
download_id,
@@ -686,26 +713,26 @@ class BDFRManager:
{
"phase": "completed",
"items_processed": processed_submissions,
"items_found": download_info["items_found"]
}
"items_found": download_info["items_found"],
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
elif hasattr(downloader, 'reddit_lists') and hasattr(downloader, 'write_entry'):
elif hasattr(downloader, "reddit_lists") and hasattr(downloader, "write_entry"):
# For Archiver, we need to intercept the archiving process
processed_items = 0
self.logger.info(f"[DEBUG] Archiver detected with reddit_lists and write_entry methods")
# Get the limit from config
limit = getattr(config, 'limit', 100)
limit = getattr(config, "limit", 100)
download_info["items_found"] = limit
self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}")
# Process items with progress tracking
self.logger.info(f"[DEBUG] Starting archiving loop...")
actual_count = 0
for generator in downloader.reddit_lists:
self.logger.info(f"[DEBUG] Processing generator for archiving...")
item_count = 0
@@ -714,14 +741,15 @@ class BDFRManager:
continue
item_count += 1
actual_count += 1
if download_info["status"] == DownloadStatus.CANCELLED.value:
self.logger.info(f"Download {download_id} was cancelled")
return
try:
# Determine item type
import praw.models
if isinstance(item, praw.models.Submission):
item_id = item.id
item_type = "submission"
@@ -729,23 +757,27 @@ class BDFRManager:
elif isinstance(item, praw.models.Comment):
item_id = item.id
item_type = "comment"
subreddit_name = item.subreddit.display_name if hasattr(item, 'subreddit') else "unknown"
subreddit_name = (
item.subreddit.display_name if hasattr(item, "subreddit") else "unknown"
)
else:
item_id = str(item)
item_type = "item"
subreddit_name = "unknown"
self.logger.info(f"[DEBUG] Archiving {item_type} {item_count}: {item_id} from r/{subreddit_name}")
self.logger.info(
f"[DEBUG] Archiving {item_type} {item_count}: {item_id} from r/{subreddit_name}"
)
# Update current item being processed
download_info["current_item"] = item_id
download_info["current_item_type"] = item_type
download_info["current_subreddit"] = subreddit_name
# Send progress update for this item
processed_items += 1
progress_percent = 20.0 + (processed_items / max(limit, 1)) * 70.0
progress_event = ProgressEvent(
"progress",
download_id,
@@ -757,11 +789,11 @@ class BDFRManager:
"current_item_type": item_type,
"current_subreddit": subreddit_name,
"items_processed": processed_items,
"items_found": download_info["items_found"]
}
"items_found": download_info["items_found"],
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
# Archive this item with retry mechanism for rate limiting
self.logger.info(f"[DEBUG] Calling write_entry for {item_id}")
try:
@@ -772,34 +804,37 @@ class BDFRManager:
error_msg = str(e)
if "429" in error_msg:
self.logger.warning(f"[DEBUG] Rate limited while archiving item {item_id}: {e}")
self.logger.warning(f"[DEBUG] Will continue with next item instead of failing entire download")
self.logger.warning(
f"[DEBUG] Will continue with next item instead of failing entire download"
)
continue
else:
# Not a rate limiting error, re-raise
raise
# Update processed count
download_info["items_processed"] = processed_items
except Exception as e:
import traceback
error_msg = str(e)
item_id = item_id if 'item_id' in locals() else "unknown"
item_id = item_id if "item_id" in locals() else "unknown"
stack_trace = traceback.format_exc()
if "429" in error_msg:
self.logger.error(f"[DEBUG] Rate limited while archiving item {item_id}: {e}")
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
download_info["status"] = DownloadStatus.FAILED.value
download_info["error"] = f"Rate limited by Reddit API: {e}"
download_info["end_time"] = datetime.now()
error_event = ProgressEvent(
"error",
download_id,
f"Rate limited by Reddit API: {e}",
None,
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"}
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"},
)
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
return
@@ -807,16 +842,18 @@ class BDFRManager:
self.logger.error(f"[DEBUG] Error archiving item {item_id}: {e}")
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
continue
self.logger.info(f"[DEBUG] Generator archived {item_count} items")
# Final progress update
if actual_count < limit:
download_info["items_found"] = actual_count
self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}")
self.logger.info(f"[DEBUG] Final progress update - archived {processed_items} out of {download_info['items_found']} found")
self.logger.info(
f"[DEBUG] Final progress update - archived {processed_items} out of {download_info['items_found']} found"
)
progress_event = ProgressEvent(
"progress",
download_id,
@@ -825,26 +862,32 @@ class BDFRManager:
{
"phase": "completed",
"items_processed": processed_items,
"items_found": download_info["items_found"]
}
"items_found": download_info["items_found"],
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
elif hasattr(downloader, 'reddit_lists') and hasattr(downloader, '_download_submission') and hasattr(downloader, 'write_entry'):
elif (
hasattr(downloader, "reddit_lists")
and hasattr(downloader, "_download_submission")
and hasattr(downloader, "write_entry")
):
# For RedditCloner (which has both download and archive capabilities)
processed_items = 0
self.logger.info(f"[DEBUG] RedditCloner detected with reddit_lists, _download_submission, and write_entry methods")
self.logger.info(
f"[DEBUG] RedditCloner detected with reddit_lists, _download_submission, and write_entry methods"
)
# Get the limit from config
limit = getattr(config, 'limit', 100)
limit = getattr(config, "limit", 100)
download_info["items_found"] = limit
self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}")
# Process submissions with progress tracking (both download and archive)
self.logger.info(f"[DEBUG] Starting cloning loop...")
actual_count = 0
for generator in downloader.reddit_lists:
self.logger.info(f"[DEBUG] Processing generator for cloning...")
submission_count = 0
@@ -853,23 +896,25 @@ class BDFRManager:
continue
submission_count += 1
actual_count += 1
if download_info["status"] == DownloadStatus.CANCELLED.value:
self.logger.info(f"Download {download_id} was cancelled")
return
try:
self.logger.info(f"[DEBUG] Cloning submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}")
self.logger.info(
f"[DEBUG] Cloning submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}"
)
# Update current item being processed
download_info["current_item"] = submission.id
download_info["current_item_type"] = "submission"
download_info["current_subreddit"] = submission.subreddit.display_name
# Send progress update for this submission
processed_items += 1
progress_percent = 20.0 + (processed_items / max(limit, 1)) * 70.0
progress_event = ProgressEvent(
"progress",
download_id,
@@ -881,11 +926,11 @@ class BDFRManager:
"current_item_type": "submission",
"current_subreddit": submission.subreddit.display_name,
"items_processed": processed_items,
"items_found": download_info["items_found"]
}
"items_found": download_info["items_found"],
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
# Clone this submission (download + archive) with retry mechanism for rate limiting
self.logger.info(f"[DEBUG] Calling _download_submission for {submission.id}")
try:
@@ -897,36 +942,43 @@ class BDFRManager:
# Check if this is a rate limiting error that should be retried
error_msg = str(e)
if "429" in error_msg:
self.logger.warning(f"[DEBUG] Rate limited while cloning submission {submission.id}: {e}")
self.logger.warning(f"[DEBUG] Will continue with next submission instead of failing entire download")
self.logger.warning(
f"[DEBUG] Rate limited while cloning submission {submission.id}: {e}"
)
self.logger.warning(
f"[DEBUG] Will continue with next submission instead of failing entire download"
)
continue
else:
# Not a rate limiting error, re-raise
raise
# Update processed count
download_info["items_processed"] = processed_items
except Exception as e:
import traceback
error_msg = str(e)
submission_id = submission.id if submission is not None else "unknown"
stack_trace = traceback.format_exc()
if "429" in error_msg:
self.logger.error(f"[DEBUG] Rate limited while cloning submission {submission_id}: {e}")
self.logger.error(
f"[DEBUG] Rate limited while cloning submission {submission_id}: {e}"
)
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
self.logger.info(f"[DEBUG] Error message contains '429': {error_msg}")
download_info["status"] = DownloadStatus.FAILED.value
download_info["error"] = f"Rate limited by Reddit API: {e}"
download_info["end_time"] = datetime.now()
error_event = ProgressEvent(
"error",
download_id,
f"Rate limited by Reddit API: {e}",
None,
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"}
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"},
)
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
return
@@ -934,16 +986,18 @@ class BDFRManager:
self.logger.error(f"[DEBUG] Error cloning submission {submission_id}: {e}")
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
continue
self.logger.info(f"[DEBUG] Generator cloned {submission_count} submissions")
# Final progress update
if actual_count < limit:
download_info["items_found"] = actual_count
self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}")
self.logger.info(f"[DEBUG] Final progress update - cloned {processed_items} out of {download_info['items_found']} found")
self.logger.info(
f"[DEBUG] Final progress update - cloned {processed_items} out of {download_info['items_found']} found"
)
progress_event = ProgressEvent(
"progress",
download_id,
@@ -952,11 +1006,11 @@ class BDFRManager:
{
"phase": "completed",
"items_processed": processed_items,
"items_found": download_info["items_found"]
}
"items_found": download_info["items_found"],
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
else:
# For other downloaders, just run normally
self.logger.info(f"[DEBUG] Using fallback downloader (no specific handler)")
@@ -964,6 +1018,7 @@ class BDFRManager:
except Exception as e:
import traceback
stack_trace = traceback.format_exc()
self.logger.error(f"Error in progress download for {download_id}: {e}")
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
@@ -971,7 +1026,7 @@ class BDFRManager:
raise
# Set up authentication if token provided
if hasattr(config, 'auth_token') and config.auth_token:
if hasattr(config, "auth_token") and config.auth_token:
self.logger.info(f"[DEBUG] Setting up authenticated Reddit instance")
# Monkey patch the create_reddit_instance method to use our token
original_create_reddit_instance = downloader.create_reddit_instance
@@ -985,11 +1040,12 @@ class BDFRManager:
# Override the reddit_instance with authenticated version
import praw
downloader.reddit_instance = praw.Reddit(
client_id=downloader.cfg_parser.get("DEFAULT", "client_id"),
client_secret=downloader.cfg_parser.get("DEFAULT", "client_secret"),
user_agent=getattr(downloader, 'user_agent', 'BDFR-Web-Interface/1.0'),
token=auth_token
user_agent=getattr(downloader, "user_agent", "BDFR-Web-Interface/1.0"),
token=auth_token,
)
downloader.authenticated = True
self.logger.info(f"[DEBUG] Created authenticated Reddit instance successfully")
@@ -1005,10 +1061,14 @@ class BDFRManager:
async def _notify_callbacks(self, callbacks: List[ProgressCallback], method_name: str, event: ProgressEvent):
"""Notify all callbacks of an event"""
self.logger.info(f"[BDFR-API] Notifying {len(callbacks)} callbacks of {method_name} event for download {event.download_id}")
self.logger.info(
f"[BDFR-API] Notifying {len(callbacks)} callbacks of {method_name} event for download {event.download_id}"
)
for callback in callbacks:
try:
logger.info(f"[BDFR-API] Invoking {method_name} on callback {callback.__class__.__name__} for download {event.download_id}")
logger.info(
f"[BDFR-API] Invoking {method_name} on callback {callback.__class__.__name__} for download {event.download_id}"
)
method = getattr(callback, method_name)
if asyncio.iscoroutinefunction(method):
await method(event)
@@ -1080,10 +1140,7 @@ class BDFRManager:
Returns:
Dictionary of download information keyed by download ID
"""
return {
download_id: self.get_download_status(download_id)
for download_id in self._active_downloads
}
return {download_id: self.get_download_status(download_id) for download_id in self._active_downloads}
def cleanup_completed(self, max_age_seconds: int = 3600) -> int:
"""
@@ -1099,7 +1156,11 @@ class BDFRManager:
to_remove = []
for download_id, download_info in self._active_downloads.items():
if download_info["status"] in [DownloadStatus.COMPLETED.value, DownloadStatus.FAILED.value, DownloadStatus.CANCELLED.value]:
if download_info["status"] in [
DownloadStatus.COMPLETED.value,
DownloadStatus.FAILED.value,
DownloadStatus.CANCELLED.value,
]:
end_time = download_info.get("end_time")
if end_time and (current_time - end_time).total_seconds() > max_age_seconds:
to_remove.append(download_id)
@@ -1124,7 +1185,7 @@ class BDFRManager:
time_filter: str = "all",
no_dupes: bool = False,
simple_check: bool = False,
progress_callbacks: Optional[List[ProgressCallback]] = None
progress_callbacks: Optional[List[ProgressCallback]] = None,
) -> str:
"""
Download content from a subreddit.
@@ -1168,12 +1229,7 @@ class BDFRManager:
config.no_dupes = no_dupes
config.simple_check = simple_check
download_id = self.create_download(
DownloadType.SUBREDDIT,
subreddit_name,
config,
progress_callbacks
)
download_id = self.create_download(DownloadType.SUBREDDIT, subreddit_name, config, progress_callbacks)
self.start_download(download_id)
return download_id
@@ -1187,7 +1243,7 @@ class BDFRManager:
saved: bool = False,
no_dupes: bool = False,
simple_check: bool = False,
progress_callbacks: Optional[List[ProgressCallback]] = None
progress_callbacks: Optional[List[ProgressCallback]] = None,
) -> str:
"""
Download content from a user.
@@ -1234,7 +1290,7 @@ class BDFRManager:
config.saved = saved
config.no_dupes = no_dupes
config.simple_check = simple_check
# Set authentication if token is available for this manager instance
if self.auth_token:
config.authenticate = True
@@ -1244,12 +1300,7 @@ class BDFRManager:
config.authenticate = False
logger.info(f"[DEBUG] No authentication token available for user download")
download_id = self.create_download(
DownloadType.USER,
username,
config,
progress_callbacks
)
download_id = self.create_download(DownloadType.USER, username, config, progress_callbacks)
self.start_download(download_id)
return download_id
@@ -1260,7 +1311,7 @@ class BDFRManager:
format_type: str = "json",
limit: Optional[int] = None,
simple_check: bool = False,
progress_callbacks: Optional[List[ProgressCallback]] = None
progress_callbacks: Optional[List[ProgressCallback]] = None,
) -> str:
"""
Archive subreddit data (metadata only, no downloads).
@@ -1298,12 +1349,7 @@ class BDFRManager:
config.limit = limit
config.format = format_type
download_id = self.create_download(
DownloadType.ARCHIVE,
subreddit_name,
config,
progress_callbacks
)
download_id = self.create_download(DownloadType.ARCHIVE, subreddit_name, config, progress_callbacks)
self.start_download(download_id)
return download_id
@@ -1315,7 +1361,7 @@ class BDFRManager:
format_type: str = "json",
no_dupes: bool = False,
simple_check: bool = False,
progress_callbacks: Optional[List[ProgressCallback]] = None
progress_callbacks: Optional[List[ProgressCallback]] = None,
) -> str:
"""
Clone subreddit (both download and archive).
@@ -1355,12 +1401,7 @@ class BDFRManager:
config.format = format_type
config.no_dupes = no_dupes
download_id = self.create_download(
DownloadType.CLONE,
subreddit_name,
config,
progress_callbacks
)
download_id = self.create_download(DownloadType.CLONE, subreddit_name, config, progress_callbacks)
self.start_download(download_id)
return download_id
@@ -1394,4 +1435,4 @@ def set_bdfr_manager(manager: BDFRManager):
manager: BDFRManager instance to use as default
"""
global _default_manager
_default_manager = manager
_default_manager = manager
bdfr/archiver.py
+3 -1
View File
@@ -52,7 +52,9 @@ class Archiver(RedditConnector):
logger.error(f"Submission {submission.id} failed to be archived due to a PRAW exception: {e}")
except prawcore.PrawcoreException as e:
if submission is not None:
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
logger.error(
f"The submission after {submission.id} failed to download due to a PRAW exception: {e}"
)
else:
logger.error(f"Download failed due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
bdfr/cloner.py
+3 -1
View File
@@ -30,7 +30,9 @@ class RedditCloner(RedditDownloader, Archiver):
logger.error(f"Submission {submission.id} failed to be cloned due to a PRAW exception: {e}")
except prawcore.PrawcoreException as e:
if submission is not None:
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
logger.error(
f"The submission after {submission.id} failed to download due to a PRAW exception: {e}"
)
else:
logger.error(f"Download failed due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
bdfr/connector.py
+11 -3
View File
@@ -414,9 +414,13 @@ class RedditConnector(metaclass=ABCMeta):
if TooManyRequests is not None and isinstance(e, TooManyRequests):
is_rate_limited = True
logger.info(f"Rate limited detected: Exception is TooManyRequests for user {user}")
elif (hasattr(e, "response") and getattr(e.response, "status_code", None) == 429) or "429" in str(e):
elif (hasattr(e, "response") and getattr(e.response, "status_code", None) == 429) or "429" in str(
e
):
is_rate_limited = True
logger.info(f"Rate limited detected: Status code 429 or '429' in error message for user {user}. Error: {e}")
logger.info(
f"Rate limited detected: Status code 429 or '429' in error message for user {user}. Error: {e}"
)
if is_rate_limited:
logger.error("Received HTTP 429 (rate limited). Propagating error to fail the download.")
@@ -441,7 +445,11 @@ class RedditConnector(metaclass=ABCMeta):
def create_file_name_formatter(self) -> FileNameFormatter:
return FileNameFormatter(
self.args.file_scheme, self.args.folder_scheme, self.args.time_format, self.args.filename_restriction_scheme, self.args.strip_unicode
self.args.file_scheme,
self.args.folder_scheme,
self.args.time_format,
self.args.filename_restriction_scheme,
self.args.strip_unicode,
)
def create_time_filter(self) -> RedditTypes.TimeType:
bdfr/downloader.py
+39 -39
View File
@@ -50,11 +50,13 @@ class RedditDownloader(RedditConnector):
hash_data = self._load_hash_list()
# Handle both old and new hash file formats
if isinstance(hash_data, dict) and 'files' in hash_data:
if isinstance(hash_data, dict) and "files" in hash_data:
# New format with enhanced structure
self.master_hash_list = {k: v['path'] for k, v in hash_data['files'].items()}
self.url_list = hash_data.get('urls', {})
logger.info(f"Loaded {len(self.master_hash_list)} hashes and {len(self.url_list)} URLs from enhanced hash file")
self.master_hash_list = {k: v["path"] for k, v in hash_data["files"].items()}
self.url_list = hash_data.get("urls", {})
logger.info(
f"Loaded {len(self.master_hash_list)} hashes and {len(self.url_list)} URLs from enhanced hash file"
)
else:
# Old format - just hashes
self.master_hash_list = hash_data
@@ -70,8 +72,10 @@ class RedditDownloader(RedditConnector):
if hash_value not in existing_hashes:
self.master_hash_list[hash_value] = file_path
logger.info(f"Loaded {len(self.master_hash_list)} total hashes "
f"({len(existing_hashes)} from file, {len(all_files_hashes) - len(existing_hashes)} new)")
logger.info(
f"Loaded {len(self.master_hash_list)} total hashes "
f"({len(existing_hashes)} from file, {len(all_files_hashes) - len(existing_hashes)} new)"
)
def download(self):
for generator in self.reddit_lists:
@@ -164,7 +168,7 @@ class RedditDownloader(RedditConnector):
self.master_hash_list[existing_file_hash] = destination
# Store URL mapping for simple-check functionality if URL is available
if hasattr(res, 'url') and self.args.simple_check:
if hasattr(res, "url") and self.args.simple_check:
self.url_list[res.url] = existing_file_hash
logger.debug(f"Added hash for existing file: {existing_file_hash}")
@@ -190,7 +194,7 @@ class RedditDownloader(RedditConnector):
resource_hash = res.hash.hexdigest()
# Simple-check: URL-based duplicate detection (fast path)
if self.args.simple_check and hasattr(res, 'url') and res.url in self.url_list:
if self.args.simple_check and hasattr(res, "url") and res.url in self.url_list:
stored_hash = self.url_list[res.url]
if stored_hash in self.master_hash_list:
logger.info(f"URL {res.url} from submission {submission.id} already downloaded (simple-check)")
@@ -233,7 +237,7 @@ class RedditDownloader(RedditConnector):
self.master_hash_list[resource_hash] = destination
# Store URL mapping for simple-check functionality
if hasattr(res, 'url') and self.args.simple_check:
if hasattr(res, "url") and self.args.simple_check:
self.url_list[res.url] = resource_hash
logger.debug(f"Hash added to master list: {resource_hash}")
@@ -252,7 +256,7 @@ class RedditDownloader(RedditConnector):
@staticmethod
def scan_existing_files(directory: Path) -> dict[str, Path]:
files = []
for (dirpath, _dirnames, filenames) in os.walk(directory):
for dirpath, _dirnames, filenames in os.walk(directory):
files.extend([Path(dirpath, file) for file in filenames])
logger.info(f"Calculating hashes for {len(files)} files")
@@ -270,14 +274,14 @@ class RedditDownloader(RedditConnector):
def _load_hash_list(self) -> dict[str, Path]:
"""Load existing hash list from .bdfr_hashes.json in download directory."""
logger.debug(f"Loading hash list from directory: {self.download_directory}")
hash_file_path = self.download_directory / '.bdfr_hashes.json'
hash_file_path = self.download_directory / ".bdfr_hashes.json"
if not hash_file_path.exists():
logger.debug(f"No existing hash file found at {hash_file_path}")
return {}
try:
with open(hash_file_path, 'r', encoding='utf-8') as f:
with open(hash_file_path, "r", encoding="utf-8") as f:
hash_data = json.load(f)
if not isinstance(hash_data, dict):
@@ -285,16 +289,16 @@ class RedditDownloader(RedditConnector):
return {}
# Handle new enhanced format
if 'files' in hash_data and isinstance(hash_data['files'], dict):
if "files" in hash_data and isinstance(hash_data["files"], dict):
# New format with enhanced structure
files_data = hash_data['files']
files_data = hash_data["files"]
loaded_hashes = {}
urls_data = hash_data.get('urls', {})
urls_data = hash_data.get("urls", {})
for hash_value, file_info in files_data.items():
if isinstance(file_info, dict) and 'path' in file_info:
if isinstance(file_info, dict) and "path" in file_info:
# New format: {"hash": {"path": "relative/path", "url": "http://..."}}
relative_path = file_info['path']
relative_path = file_info["path"]
absolute_path = self.download_directory / relative_path
if absolute_path.exists():
loaded_hashes[hash_value] = absolute_path
@@ -302,8 +306,8 @@ class RedditDownloader(RedditConnector):
logger.debug(f"File {absolute_path} from hash file no longer exists")
# Load URL mapping for simple-check
if 'url' in file_info and file_info['url']:
self.url_list[file_info['url']] = hash_value
if "url" in file_info and file_info["url"]:
self.url_list[file_info["url"]] = hash_value
elif isinstance(file_info, str):
# Legacy format within new structure: {"hash": "relative/path"}
absolute_path = self.download_directory / file_info
@@ -337,30 +341,30 @@ class RedditDownloader(RedditConnector):
def _save_hash_list(self) -> None:
"""Save current hash list to .bdfr_hashes.json in download directory using atomic write."""
hash_file_path = self.download_directory / '.bdfr_hashes.json'
hash_file_path = self.download_directory / ".bdfr_hashes.json"
# Build enhanced data structure for new format
if self.args.simple_check:
# New enhanced format with URLs and metadata
hash_data = {
'files': {},
'urls': self.url_list.copy(),
'metadata': {
'version': '2.0',
'created_with': 'simple_check' if self.args.simple_check else 'standard',
'url_count': len(self.url_list),
'hash_count': len(self.master_hash_list)
}
"files": {},
"urls": self.url_list.copy(),
"metadata": {
"version": "2.0",
"created_with": "simple_check" if self.args.simple_check else "standard",
"url_count": len(self.url_list),
"hash_count": len(self.master_hash_list),
},
}
# Convert absolute paths to relative paths for portability
for hash_value, absolute_path in self.master_hash_list.items():
try:
relative_path = absolute_path.relative_to(self.download_directory)
hash_data['files'][hash_value] = {
'path': str(relative_path),
'url': next((url for url, h in self.url_list.items() if h == hash_value), None),
'check_method': 'hash'
hash_data["files"][hash_value] = {
"path": str(relative_path),
"url": next((url for url, h in self.url_list.items() if h == hash_value), None),
"check_method": "hash",
}
except ValueError:
# File is not relative to download directory, skip it
@@ -381,17 +385,13 @@ class RedditDownloader(RedditConnector):
# Atomic write: write to temporary file first, then rename
try:
with tempfile.NamedTemporaryFile(
mode='w',
dir=self.download_directory,
suffix='.tmp',
delete=False,
encoding='utf-8'
mode="w", dir=self.download_directory, suffix=".tmp", delete=False, encoding="utf-8"
) as temp_file:
json.dump(hash_data, temp_file, indent=2)
temp_file_path = temp_file.name
# Atomic rename
if os.name == 'nt': # Windows
if os.name == "nt": # Windows
# On Windows, we need to remove the target file first if it exists
if hash_file_path.exists():
hash_file_path.unlink()
@@ -407,7 +407,7 @@ class RedditDownloader(RedditConnector):
logger.error(f"Unexpected error saving hash file {hash_file_path}: {e}")
# Clean up temp file if it still exists
try:
if 'temp_file_path' in locals():
if "temp_file_path" in locals():
os.unlink(temp_file_path)
except (OSError, IOError):
pass
bdfr/examples/api_usage.py
+24 -50
View File
@@ -19,20 +19,16 @@ sys.path.insert(0, str(Path(__file__).parent.parent))
from bdfr.api import (
BDFRManager,
DownloadType,
DownloadStatus,
ProgressEvent,
ProgressCallback,
DownloadType,
LoggingCallback,
get_bdfr_manager
ProgressCallback,
ProgressEvent,
get_bdfr_manager,
)
# Configure logging
logging.basicConfig(
level=logging.INFO,
format='[%(asctime)s] %(levelname)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S'
)
logging.basicConfig(level=logging.INFO, format="[%(asctime)s] %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
logger = logging.getLogger(__name__)
@@ -53,7 +49,7 @@ class WebSocketCallback(ProgressCallback):
async def on_error(self, event: ProgressEvent):
"""Send error update to WebSocket"""
print(f"❌ [{self.websocket_id}] ERROR: {event.message}")
if event.data.get('exception'):
if event.data.get("exception"):
print(f" Exception: {event.data['exception']}")
async def on_completed(self, event: ProgressEvent):
@@ -95,9 +91,9 @@ async def example_basic_usage():
# Create a download for a subreddit
download_id = manager.download_subreddit(
"python", # subreddit name
limit=10, # download 10 posts
sort="hot", # sort by hot
no_dupes=True # avoid duplicates
limit=10, # download 10 posts
sort="hot", # sort by hot
no_dupes=True, # avoid duplicates
)
print(f"📋 Created download with ID: {download_id}")
@@ -111,7 +107,7 @@ async def example_basic_usage():
print(f"📊 Status: {status['status']} | Progress: {int(round(status['progress']))}%")
if status['status'] in ['completed', 'failed', 'cancelled']:
if status["status"] in ["completed", "failed", "cancelled"]:
print(f"🏁 Download finished with status: {status['status']}")
break
@@ -126,11 +122,7 @@ async def example_advanced_usage():
print("=" * 50)
# Create custom callbacks
callbacks = [
LoggingCallback("web_interface"),
WebSocketCallback("user_123"),
DatabaseCallback()
]
callbacks = [LoggingCallback("web_interface"), WebSocketCallback("user_123"), DatabaseCallback()]
# Create manager with custom download directory
manager = BDFRManager("./custom_downloads")
@@ -140,11 +132,7 @@ async def example_advanced_usage():
download_ids = []
for subreddit in subreddits:
download_id = manager.create_download(
DownloadType.SUBREDDIT,
subreddit,
progress_callbacks=callbacks
)
download_id = manager.create_download(DownloadType.SUBREDDIT, subreddit, progress_callbacks=callbacks)
# Start the download
manager.start_download(download_id)
@@ -164,7 +152,7 @@ async def example_advanced_usage():
print(f"📊 {download_id}: {status['status']} ({int(round(status['progress']))}%)")
if status['status'] in ['completed', 'failed', 'cancelled']:
if status["status"] in ["completed", "failed", "cancelled"]:
print(f"🏁 Download {download_id} finished")
download_ids.remove(download_id)
else:
@@ -187,11 +175,7 @@ async def example_user_download():
# Download user's submitted posts
download_id = manager.download_user(
"testuser", # username
limit=25, # 25 posts
submitted=True,
upvoted=False,
saved=False
"testuser", limit=25, submitted=True, upvoted=False, saved=False # username # 25 posts
)
print(f"📋 Created user download: {download_id}")
@@ -205,7 +189,7 @@ async def example_user_download():
print(f"📊 Status: {status['status']} | Progress: {int(round(status['progress']))}%")
if status['status'] in ['completed', 'failed']:
if status["status"] in ["completed", "failed"]:
break
await asyncio.sleep(2)
@@ -221,11 +205,7 @@ async def example_archive_operation():
manager = BDFRManager("./archives")
# Archive subreddit data (metadata only)
download_id = manager.archive_subreddit(
"dataisbeautiful",
format_type="json",
limit=50
)
download_id = manager.archive_subreddit("dataisbeautiful", format_type="json", limit=50)
print(f"📋 Created archive operation: {download_id}")
@@ -238,7 +218,7 @@ async def example_archive_operation():
print(f"📊 Archive status: {status['status']} | Progress: {int(round(status['progress']))}%")
if status['status'] in ['completed', 'failed']:
if status["status"] in ["completed", "failed"]:
print(f"🏁 Archive finished with status: {status['status']}")
break
@@ -265,11 +245,7 @@ async def example_web_integration():
callback = WebSocketCallback(f"ws_{user_id}")
# Create and start download
download_id = self.manager.download_subreddit(
subreddit,
limit=limit,
progress_callbacks=[callback]
)
download_id = self.manager.download_subreddit(subreddit, limit=limit, progress_callbacks=[callback])
# Track for this user session
if user_id not in self.active_sessions:
@@ -279,7 +255,7 @@ async def example_web_integration():
return {
"success": True,
"download_id": download_id,
"message": f"Started download of r/{subreddit} (limit: {limit})"
"message": f"Started download of r/{subreddit} (limit: {limit})",
}
async def get_user_downloads(self, user_id: str):
@@ -329,7 +305,7 @@ async def example_web_integration():
# Cancel one download
if user_downloads:
cancel_result = await app.cancel_user_download(user_id, user_downloads[0]['id'])
cancel_result = await app.cancel_user_download(user_id, user_downloads[0]["id"])
print(f"Cancel result: {cancel_result}")
return len(user_downloads)
@@ -343,10 +319,7 @@ async def example_error_handling():
manager = BDFRManager("./test_downloads")
# Try to download from a non-existent subreddit
download_id = manager.download_subreddit(
"this_subreddit_does_not_exist",
limit=5
)
download_id = manager.download_subreddit("this_subreddit_does_not_exist", limit=5)
print(f"📋 Created download for non-existent subreddit: {download_id}")
@@ -359,7 +332,7 @@ async def example_error_handling():
print(f"📊 Status: {status['status']}")
if status['status'] == 'failed':
if status["status"] == "failed":
print(f"🏁 Download failed as expected: {status.get('error', 'Unknown error')}")
break
@@ -401,9 +374,10 @@ async def main():
except Exception as e:
print(f"\n❌ Error running examples: {e}")
import traceback
traceback.print_exc()
if __name__ == "__main__":
# Run the examples
asyncio.run(main())
asyncio.run(main())
bdfr/file_name_formatter.py
+19 -3
View File
@@ -245,10 +245,26 @@ class FileNameFormatter:
if ord(char) < 0x80:
result.append(char)
# Keep common Unicode letters, numbers, and punctuation
elif unicodedata.category(char) in ['Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Ps', 'Pe', 'Pi', 'Pf', 'Po']:
elif unicodedata.category(char) in [
"Lu",
"Ll",
"Lt",
"Lm",
"Lo",
"Nd",
"Nl",
"No",
"Pc",
"Pd",
"Ps",
"Pe",
"Pi",
"Pf",
"Po",
]:
result.append(char)
# Strip emoji, symbols, and other special characters that cause 8.3 names
elif unicodedata.category(char).startswith(('S', 'So', 'Sk', 'Sm')): # Symbols
elif unicodedata.category(char).startswith(("S", "So", "Sk", "Sm")): # Symbols
continue
elif ord(char) > 0x1F000: # High Unicode ranges often contain emoji
continue
@@ -256,7 +272,7 @@ class FileNameFormatter:
# Keep other Unicode characters that are generally safe
result.append(char)
return ''.join(result)
return "".join(result)
@staticmethod
def _strip_emojis(input_string: str) -> str:
bdfr/resource.py
+22 -20
View File
@@ -24,11 +24,11 @@ class Resource:
self.content: Optional[bytes] = None
self.url = url
self.hash: Optional[_hashlib.HASH] = None
# Log the original extension before normalization
if extension:
logger.debug(f"Resource constructor received extension: '{extension}' for URL: {url}")
self.extension = self._normalize_extension(extension)
self.download_function = download_function
if not self.extension:
@@ -72,14 +72,14 @@ class Resource:
if self.url.startswith("https://www.reddit.com/media"):
logger.debug(f"Detected Reddit media URL: {self.url}")
parsed_url = urllib.parse.urlparse(self.url)
url_param = urllib.parse.parse_qs(parsed_url.query).get('url', [None])[0]
url_param = urllib.parse.parse_qs(parsed_url.query).get("url", [None])[0]
if url_param:
decoded_url = urllib.parse.unquote(url_param)
logger.debug(f"Reddit media URL decoded to: {decoded_url}")
stripped_url = urllib.parse.urlsplit(decoded_url).path
# Also handle preview.redd.it URLs which might not have extensions
elif "preview.redd.it" in self.url and not stripped_url.endswith(('.jpg', '.jpeg', '.png', '.gif', '.webp')):
elif "preview.redd.it" in self.url and not stripped_url.endswith((".jpg", ".jpeg", ".png", ".gif", ".webp")):
logger.debug(f"Detected preview.redd.it URL without extension: {self.url}")
# For preview URLs, try to infer from common patterns or add fallback logic
@@ -92,7 +92,7 @@ class Resource:
logger.warning(f"Could not determine extension for URL: {self.url} (path: {stripped_url})")
# As a last resort, if we have content, try to detect by magic numbers
if hasattr(self, 'content') and self.content:
if hasattr(self, "content") and self.content:
detected = self._detect_extension_by_content()
return self._normalize_extension(detected) if detected else None
@@ -104,21 +104,21 @@ class Resource:
return None
# Check for common image formats
if self.content.startswith(b'\xFF\xD8\xFF'):
if self.content.startswith(b"\xff\xd8\xff"):
logger.debug(f"Detected JPEG by magic number for URL: {self.url}")
return '.jpg'
elif self.content.startswith(b'\x89PNG\r\n\x1a\n'):
return ".jpg"
elif self.content.startswith(b"\x89PNG\r\n\x1a\n"):
logger.debug(f"Detected PNG by magic number for URL: {self.url}")
return '.png'
elif self.content.startswith(b'GIF87a') or self.content.startswith(b'GIF89a'):
return ".png"
elif self.content.startswith(b"GIF87a") or self.content.startswith(b"GIF89a"):
logger.debug(f"Detected GIF by magic number for URL: {self.url}")
return '.gif'
elif self.content.startswith(b'RIFF') and self.content[8:12] == b'WEBP':
return ".gif"
elif self.content.startswith(b"RIFF") and self.content[8:12] == b"WEBP":
logger.debug(f"Detected WebP by magic number for URL: {self.url}")
return '.webp'
elif self.content.startswith(b'BM'):
return ".webp"
elif self.content.startswith(b"BM"):
logger.debug(f"Detected BMP by magic number for URL: {self.url}")
return '.bmp'
return ".bmp"
logger.debug(f"Could not detect file type by magic number for URL: {self.url}")
return None
@@ -127,15 +127,17 @@ class Resource:
"""Normalize extension to lowercase for consistency"""
if not extension:
return None
original = extension
# Ensure extension starts with a dot
if not extension.startswith('.'):
extension = '.' + extension
if not extension.startswith("."):
extension = "." + extension
normalized = extension.lower()
if original != normalized:
logger.info(f"Extension normalization: '{original}' -> '{normalized}' for URL: {self.url if hasattr(self, 'url') else 'unknown'}")
logger.info(
f"Extension normalization: '{original}' -> '{normalized}' for URL: {self.url if hasattr(self, 'url') else 'unknown'}"
)
return normalized
@staticmethod
bdfr/site_downloaders/download_factory.py
+1
View File
@@ -25,6 +25,7 @@ class DownloadFactory:
@staticmethod
def pull_lever(url: str) -> type[BaseDownloader]:
import logging
logger = logging.getLogger(__name__)
sanitised_url = DownloadFactory.sanitise_url(url).lower()
tests/test_duplicate_fix.py
+7 -5
View File
@@ -4,14 +4,15 @@ Test script to verify that the duplicate folder creation fix works correctly.
This script simulates the scenario where duplicate posts would previously create empty folders.
"""
import tempfile
import shutil
from pathlib import Path
from unittest.mock import MagicMock
# Add the bdfr module to the path
import sys
sys.path.insert(0, '.')
import tempfile
from pathlib import Path
from unittest.mock import MagicMock
sys.path.insert(0, ".")
from bdfr.configuration import Configuration
from bdfr.connector import RedditConnector
@@ -69,6 +70,7 @@ def test_duplicate_folder_creation_fix():
# Mock the download factory
import bdfr.site_downloaders.download_factory as df
original_pull_lever = df.DownloadFactory.pull_lever
df.DownloadFactory.pull_lever = MagicMock(return_value=mock_downloader_class)
@@ -93,4 +95,4 @@ def test_duplicate_folder_creation_fix():
if __name__ == "__main__":
test_duplicate_folder_creation_fix()
print("All tests passed! The duplicate folder creation fix is working correctly.")
print("All tests passed! The duplicate folder creation fix is working correctly.")
tests/test_extension_case_normalization.py
+11 -5
View File
@@ -3,8 +3,10 @@
Test extension case normalization functionality
"""
import pytest
from unittest.mock import MagicMock
import pytest
from bdfr.resource import Resource
@@ -43,7 +45,9 @@ class TestExtensionNormalization:
mock_submission.id = "test123"
resource = Resource(mock_submission, url, lambda: None)
assert resource.extension == expected, f"Reddit media URL {url} should normalize to {expected}, got {resource.extension}"
assert (
resource.extension == expected
), f"Reddit media URL {url} should normalize to {expected}, got {resource.extension}"
def test_constructor_extensions_normalized(self):
"""Test that extensions passed to constructor are normalized"""
@@ -66,7 +70,9 @@ class TestExtensionNormalization:
mock_submission.id = "test123"
resource = Resource(mock_submission, "https://example.com/test", lambda: None, input_ext)
assert resource.extension == expected, f"Constructor extension {input_ext} should normalize to {expected}, got {resource.extension}"
assert (
resource.extension == expected
), f"Constructor extension {input_ext} should normalize to {expected}, got {resource.extension}"
def test_magic_number_detection_normalized(self):
"""Test that magic number detection returns normalized extensions"""
@@ -74,7 +80,7 @@ class TestExtensionNormalization:
mock_submission.id = "test123"
# Test JPEG magic number detection
jpeg_content = b'\xFF\xD8\xFF' + b'0' * 100 # JPEG magic number
jpeg_content = b"\xff\xd8\xff" + b"0" * 100 # JPEG magic number
resource = Resource(mock_submission, "https://example.com/no-extension", lambda params: jpeg_content)
resource.download() # Trigger content-based detection
assert resource.extension == ".jpg", f"Magic number detection should return .jpg, got {resource.extension}"
assert resource.extension == ".jpg", f"Magic number detection should return .jpg, got {resource.extension}"
tests/test_extension_debug.py
+6 -8
View File
@@ -3,13 +3,16 @@
Test script to debug file extension detection issues
"""
import sys
import os
import sys
sys.path.insert(0, os.path.dirname(__file__))
from unittest.mock import MagicMock
from bdfr.resource import Resource
def test_extension_detection():
"""Test extension detection with various URL patterns"""
@@ -18,26 +21,20 @@ def test_extension_detection():
("https://example.com/image.jpg", ".jpg"),
("https://example.com/video.mp4", ".mp4"),
("https://files.example.com/document.pdf", ".pdf"),
# URLs without extensions
("https://example.com/api/data", None),
("https://example.com/path/without/extension", None),
# URLs with query parameters
("https://example.com/image.jpg?size=large", ".jpg"),
("https://example.com/video.mp4?utm_source=test", ".mp4"),
# URLs with fragments
("https://example.com/image.png#section", ".png"),
# Complex paths
("https://imgur.com/a/gallery123", None),
("https://reddit.com/r/test/abc123_def456_789", None),
# Edge cases that might cause weird names
("https://example.com/L7SW9E~G", None),
("https://example.com/temp/file", None),
# Reddit media URLs (the actual issue)
("https://www.reddit.com/media?url=https%3A%2F%2Fi.redd.it%2Fr2mv10i4vkfd1.jpeg", ".jpeg"),
("https://i.redd.it/r2mv10i4vkfd1.jpeg", ".jpeg"),
@@ -60,5 +57,6 @@ def test_extension_detection():
print(f"Match: {'YES' if resource.extension == expected else 'NO'}")
print("-" * 40)
if __name__ == "__main__":
test_extension_detection()
test_extension_detection()
tests/test_file_locking_fix.py
+12 -9
View File
@@ -7,16 +7,18 @@ This script simulates the scenario where a download fails and then tries to redo
import asyncio
import logging
import os
# Add the bdfr module to the path
import sys
import tempfile
import time
from pathlib import Path
# Add the bdfr module to the path
import sys
sys.path.insert(0, str(Path(__file__).parent))
from bdfr.api import BDFRManager, DownloadType, LoggingCallback, ProgressEvent
class TestProgressCallback(LoggingCallback):
"""Test callback that simulates a failure"""
@@ -36,6 +38,7 @@ class TestProgressCallback(LoggingCallback):
self.events.append(event)
await super().on_completed(event)
def test_file_locking_fix():
"""Test that the file locking issue is resolved"""
print("Testing file locking fix...")
@@ -53,7 +56,7 @@ def test_file_locking_fix():
download_id1 = manager.create_download(
DownloadType.USER,
"test_user_12345", # This user doesn't exist, should fail
progress_callbacks=[TestProgressCallback()]
progress_callbacks=[TestProgressCallback()],
)
# Start the download (it should fail)
@@ -71,7 +74,7 @@ def test_file_locking_fix():
download_id2 = manager.create_download(
DownloadType.USER,
"test_user_67890", # This user also doesn't exist, should fail
progress_callbacks=[TestProgressCallback()]
progress_callbacks=[TestProgressCallback()],
)
# Start the second download
@@ -100,7 +103,7 @@ def test_file_locking_fix():
print(f" - {log_file.name}")
# Check if file is accessible (not locked)
try:
with open(log_file, 'r') as f:
with open(log_file, "r") as f:
content = f.read()
print(f" SUCCESS: Log file is accessible ({len(content)} characters)")
except PermissionError:
@@ -112,9 +115,7 @@ def test_file_locking_fix():
# Test 4: Try to create a third download to ensure cleanup worked
print("\n4. Creating third download to verify cleanup...")
download_id3 = manager.create_download(
DownloadType.USER,
"test_user_cleanup",
progress_callbacks=[TestProgressCallback()]
DownloadType.USER, "test_user_cleanup", progress_callbacks=[TestProgressCallback()]
)
success3 = manager.start_download(download_id3)
@@ -132,6 +133,7 @@ def test_file_locking_fix():
print("\nSUCCESS: All tests passed! File locking issue appears to be fixed.")
return True
if __name__ == "__main__":
try:
success = test_file_locking_fix()
@@ -144,5 +146,6 @@ if __name__ == "__main__":
except Exception as e:
print(f"\nTest failed with exception: {e}")
import traceback
traceback.print_exc()
sys.exit(1)
sys.exit(1)
tests/test_file_name_formatter.py
+2 -2
View File
@@ -538,7 +538,7 @@ def test_strip_unicode_chars(input_string: str, expected: str):
def test_unicode_stripping_enabled(submission: MagicMock):
"""Test that Unicode stripping is applied when enabled"""
submission.title = 'Test 💕 emoji'
submission.title = "Test 💕 emoji"
formatter = FileNameFormatter("{TITLE}", "", "", strip_unicode=True)
result = formatter._format_name(submission, "{TITLE}")
assert "💕" not in result
@@ -547,7 +547,7 @@ def test_unicode_stripping_enabled(submission: MagicMock):
def test_unicode_stripping_disabled(submission: MagicMock):
"""Test that Unicode stripping is not applied when disabled"""
submission.title = 'Test 💕 emoji'
submission.title = "Test 💕 emoji"
formatter = FileNameFormatter("{TITLE}", "", "", strip_unicode=False)
result = formatter._format_name(submission, "{TITLE}")
assert "💕" in result
tests/test_hash_persistence.py
+43 -41
View File
@@ -3,14 +3,15 @@
Test script to verify hash persistence functionality.
"""
import json
import tempfile
import shutil
from pathlib import Path
from unittest.mock import Mock
# Import the necessary modules
import sys
sys.path.insert(0, '/Users/Daniel/Documents/GitHub/bulk-downloader-for-reddit')
import tempfile
from pathlib import Path
from unittest.mock import Mock
sys.path.insert(0, "/Users/Daniel/Documents/GitHub/bulk-downloader-for-reddit")
from bdfr.configuration import Configuration
from bdfr.downloader import RedditDownloader
@@ -59,7 +60,7 @@ def test_hash_persistence():
# Test 2: Save empty hash list
print("Test 2: Saving empty hash list")
downloader._save_hash_list()
hash_file = temp_path / '.bdfr_hashes.json'
hash_file = temp_path / ".bdfr_hashes.json"
assert hash_file.exists(), "Hash file should be created even when empty"
print("PASS Passed")
@@ -71,18 +72,18 @@ def test_hash_persistence():
# Test 4: Add some test data and save
print("Test 4: Adding test data and saving")
test_file = temp_path / 'test.txt'
test_file = temp_path / "test.txt"
test_file.write_text("test content")
downloader.master_hash_list['test_hash_123'] = test_file
downloader.master_hash_list["test_hash_123"] = test_file
downloader._save_hash_list()
# Verify the saved JSON structure
with open(hash_file, 'r') as f:
with open(hash_file, "r") as f:
saved_data = json.load(f)
assert 'test_hash_123' in saved_data, "Test hash should be in saved data"
assert saved_data['test_hash_123'] == 'test.txt', f"Expected 'test.txt', got {saved_data['test_hash_123']}"
assert "test_hash_123" in saved_data, "Test hash should be in saved data"
assert saved_data["test_hash_123"] == "test.txt", f"Expected 'test.txt', got {saved_data['test_hash_123']}"
print("PASS Passed")
# Test 5: Load hash list and verify data is restored
@@ -100,13 +101,15 @@ def test_hash_persistence():
loaded_hash_list = new_downloader._load_hash_list()
assert len(loaded_hash_list) == 1, f"Expected 1 hash, got {len(loaded_hash_list)}"
assert 'test_hash_123' in loaded_hash_list, "Test hash should be loaded"
assert loaded_hash_list['test_hash_123'] == test_file, f"File path should match: {loaded_hash_list['test_hash_123']} != {test_file}"
assert "test_hash_123" in loaded_hash_list, "Test hash should be loaded"
assert (
loaded_hash_list["test_hash_123"] == test_file
), f"File path should match: {loaded_hash_list['test_hash_123']} != {test_file}"
print("PASS Passed")
# Test 6: Test corrupted hash file handling
print("Test 6: Testing corrupted hash file handling")
with open(hash_file, 'w') as f:
with open(hash_file, "w") as f:
f.write("invalid json content")
corrupted_downloader = RedditDownloader.__new__(RedditDownloader)
@@ -122,7 +125,9 @@ def test_hash_persistence():
# Should handle corrupted file gracefully and return empty dict
corrupted_hash_list = corrupted_downloader._load_hash_list()
assert len(corrupted_hash_list) == 0, f"Expected empty hash list for corrupted file, got {len(corrupted_hash_list)}"
assert (
len(corrupted_hash_list) == 0
), f"Expected empty hash list for corrupted file, got {len(corrupted_hash_list)}"
print("PASS Passed")
print("\nAll tests passed! Hash persistence functionality is working correctly.")
@@ -174,7 +179,7 @@ def test_simple_check_functionality():
# Test 2: Add test data and save with simple_check format
print("Test 2: Adding test data and saving with simple_check format")
test_file = temp_path / 'test.txt'
test_file = temp_path / "test.txt"
test_file.write_text("test content")
test_url = "https://example.com/test.txt"
test_hash = "test_hash_123"
@@ -185,16 +190,16 @@ def test_simple_check_functionality():
downloader._save_hash_list()
# Verify the saved JSON structure has enhanced format
with open(temp_path / '.bdfr_hashes.json', 'r') as f:
with open(temp_path / ".bdfr_hashes.json", "r") as f:
saved_data = json.load(f)
assert 'files' in saved_data, "Enhanced format should have 'files' section"
assert 'urls' in saved_data, "Enhanced format should have 'urls' section"
assert 'metadata' in saved_data, "Enhanced format should have 'metadata' section"
assert test_hash in saved_data['files'], "Test hash should be in files section"
assert test_url in saved_data['urls'], "Test URL should be in urls section"
assert saved_data['metadata']['version'] == '2.0', "Version should be 2.0"
assert saved_data['metadata']['created_with'] == 'simple_check', "Should be created with simple_check"
assert "files" in saved_data, "Enhanced format should have 'files' section"
assert "urls" in saved_data, "Enhanced format should have 'urls' section"
assert "metadata" in saved_data, "Enhanced format should have 'metadata' section"
assert test_hash in saved_data["files"], "Test hash should be in files section"
assert test_url in saved_data["urls"], "Test URL should be in urls section"
assert saved_data["metadata"]["version"] == "2.0", "Version should be 2.0"
assert saved_data["metadata"]["created_with"] == "simple_check", "Should be created with simple_check"
print("PASS")
# Test 3: Load hash list and verify URL mapping is restored
@@ -228,7 +233,7 @@ def test_simple_check_functionality():
mock_resource.hash.hexdigest.return_value = test_hash
# Create a mock destination that exists
mock_destination = temp_path / 'existing_file.txt'
mock_destination = temp_path / "existing_file.txt"
mock_destination.parent.mkdir(parents=True, exist_ok=True)
mock_destination.write_text("existing content")
@@ -260,17 +265,14 @@ def test_backward_compatibility():
temp_path = Path(temp_dir)
# Create old-format hash file manually
(temp_path / 'relative' / 'path').mkdir(parents=True, exist_ok=True)
(temp_path / 'relative' / 'path' / 'file1.txt').write_text("content1")
(temp_path / 'relative' / 'path' / 'file2.txt').write_text("content2")
(temp_path / "relative" / "path").mkdir(parents=True, exist_ok=True)
(temp_path / "relative" / "path" / "file1.txt").write_text("content1")
(temp_path / "relative" / "path" / "file2.txt").write_text("content2")
old_hash_data = {
"hash1": "relative/path/file1.txt",
"hash2": "relative/path/file2.txt"
}
old_hash_data = {"hash1": "relative/path/file1.txt", "hash2": "relative/path/file2.txt"}
hash_file = temp_path / '.bdfr_hashes.json'
with open(hash_file, 'w') as f:
hash_file = temp_path / ".bdfr_hashes.json"
with open(hash_file, "w") as f:
json.dump(old_hash_data, f)
# Create downloader and load old format
@@ -294,21 +296,21 @@ def test_backward_compatibility():
print("PASS - Old format loaded correctly")
# Test saving in new format
(temp_path / 'another').mkdir(parents=True, exist_ok=True)
test_file = temp_path / 'another' / 'new_file.txt'
(temp_path / "another").mkdir(parents=True, exist_ok=True)
test_file = temp_path / "another" / "new_file.txt"
test_file.write_text("new content")
downloader.master_hash_list["new_hash"] = test_file
downloader._save_hash_list()
# Verify new format was created
with open(hash_file, 'r') as f:
with open(hash_file, "r") as f:
new_data = json.load(f)
assert 'files' in new_data, "New format should have 'files' section"
assert 'urls' in new_data, "New format should have 'urls' section"
assert 'metadata' in new_data, "New format should have 'metadata' section"
assert new_data['metadata']['version'] == '2.0', "Should be version 2.0"
assert "files" in new_data, "New format should have 'files' section"
assert "urls" in new_data, "New format should have 'urls' section"
assert "metadata" in new_data, "New format should have 'metadata' section"
assert new_data["metadata"]["version"] == "2.0", "Should be version 2.0"
print("PASS - Old format upgraded to new format correctly")
@@ -318,4 +320,4 @@ def test_backward_compatibility():
if __name__ == "__main__":
test_hash_persistence()
test_simple_check_functionality()
test_backward_compatibility()
test_backward_compatibility()
tests/test_user_folder_structure.py
+31 -35
View File
@@ -14,10 +14,11 @@ import tempfile
from pathlib import Path
# Set UTF-8 encoding for Windows console
if sys.platform == 'win32':
if sys.platform == "win32":
import codecs
sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer, 'strict')
sys.stderr = codecs.getwriter('utf-8')(sys.stderr.buffer, 'strict')
sys.stdout = codecs.getwriter("utf-8")(sys.stdout.buffer, "strict")
sys.stderr = codecs.getwriter("utf-8")(sys.stderr.buffer, "strict")
from bdfr.api import BDFRManager, DownloadType
from bdfr.configuration import Configuration
@@ -26,27 +27,24 @@ from bdfr.configuration import Configuration
def test_subreddit_directory_structure():
"""Test that subreddit downloads use correct directory structure"""
print("\n=== Testing Subreddit Directory Structure ===")
with tempfile.TemporaryDirectory() as tmpdir:
manager = BDFRManager(download_directory=tmpdir)
# Create a download for a subreddit
download_id = manager.create_download(
DownloadType.SUBREDDIT,
"test_subreddit"
)
download_id = manager.create_download(DownloadType.SUBREDDIT, "test_subreddit")
download_info = manager.get_download_status(download_id)
config = download_info["config"]
expected_dir = str(Path(tmpdir))
actual_dir = config.directory
print(f"Expected directory: {expected_dir}")
print(f"Actual directory: {actual_dir}")
print(f"Subreddit config: {config.subreddit}")
print(f"Folder scheme: {config.folder_scheme}")
assert actual_dir == expected_dir, f"Subreddit directory mismatch!"
print("[OK] Subreddit directory structure is correct")
print(f" Files will be saved to: {actual_dir}/{{SUBREDDIT}}/{{files}}")
@@ -55,28 +53,25 @@ def test_subreddit_directory_structure():
def test_user_directory_structure():
"""Test that user downloads use correct directory structure"""
print("\n=== Testing User Directory Structure ===")
with tempfile.TemporaryDirectory() as tmpdir:
manager = BDFRManager(download_directory=tmpdir)
# Create a download for a user
username = "test_user"
download_id = manager.create_download(
DownloadType.USER,
username
)
download_id = manager.create_download(DownloadType.USER, username)
download_info = manager.get_download_status(download_id)
config = download_info["config"]
expected_dir = str(Path(tmpdir) / username)
actual_dir = config.directory
print(f"Expected directory: {expected_dir}")
print(f"Actual directory: {actual_dir}")
print(f"User config: {config.user}")
print(f"Folder scheme: {config.folder_scheme}")
assert actual_dir == expected_dir, f"User directory mismatch!"
print("[OK] User directory structure is correct")
print(f" Files will be saved to: {actual_dir}/{{SUBREDDIT}}/{{files}}")
@@ -85,26 +80,26 @@ def test_user_directory_structure():
def test_convenience_method():
"""Test the convenience method download_user()"""
print("\n=== Testing download_user() Convenience Method ===")
with tempfile.TemporaryDirectory() as tmpdir:
manager = BDFRManager(download_directory=tmpdir)
# Don't actually start the download, just check the config
username = "convenience_test_user"
# Create config manually like the convenience method does
config = Configuration()
user_directory = Path(tmpdir) / username
config.directory = str(user_directory)
config.user = [username]
expected_dir = str(Path(tmpdir) / username)
actual_dir = config.directory
print(f"Expected directory: {expected_dir}")
print(f"Actual directory: {actual_dir}")
print(f"User config: {config.user}")
assert actual_dir == expected_dir, f"Convenience method directory mismatch!"
print("[OK] Convenience method directory structure is correct")
@@ -118,7 +113,7 @@ def demonstrate_folder_structure():
print(" ├── file1.jpg")
print(" ├── file2.png")
print(" └── file3.mp4")
print("\nWhen downloading from a USER 'spez' who posts to multiple subreddits:")
print(" downloads/")
print(" └── spez/")
@@ -129,7 +124,7 @@ def demonstrate_folder_structure():
print(" │ └── file3.jpg")
print(" └── pics/")
print(" └── file4.png")
print("\n[OK] This structure allows:")
print(" 1. Easy identification of user-specific downloads")
print(" 2. Organization by subreddit within each user folder")
@@ -140,22 +135,23 @@ if __name__ == "__main__":
print("=" * 60)
print("Testing User Folder Structure Changes")
print("=" * 60)
try:
test_subreddit_directory_structure()
test_user_directory_structure()
test_convenience_method()
demonstrate_folder_structure()
print("\n" + "=" * 60)
print("[SUCCESS] All tests passed!")
print("=" * 60)
except AssertionError as e:
print(f"\n[FAIL] Test failed: {e}")
exit(1)
except Exception as e:
print(f"\n[ERROR] Unexpected error: {e}")
import traceback
traceback.print_exc()
exit(1)
exit(1)
web_interface/app/main.py
-1
View File
@@ -454,7 +454,6 @@ async def create_download_with_bdfr_api(download_type: str, name: str, **kwargs)
return download_id
# Get global auth token if available
global _bdfr_auth_token
auth_token = _bdfr_auth_token
logger.info(f"[DEBUG] Global auth token available: {auth_token is not None}")
logger.info(f"[DEBUG] Auth token for BDFR: {auth_token[:10]}..." if auth_token else "None")