ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reformatting
formatting_check / formatting_check (push) Failing after 3s
Details
Python Test / test (.ps1, windows-latest, 3.9) (push) Has been cancelled
Details
Python Test / test (.sh, macos-latest, 3.9) (push) Has been cancelled
Details
Python Test / test (.sh, ubuntu-latest, 3.9) (push) Has been cancelled
Details

Browse Source
This commit is contained in:
ModerateWinGuy
2026-07-14 21:32:55 +12:00
parent 3d0658b483
commit 0157f462cc
17 changed files with 439 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bdfr/api.py
+194 -153
View File
@@ -25,12 +25,12 @@ from typing import Any, Dict, List, Optional, Union
import prawcore
from bdfr import exceptions as errors
from bdfr.archiver import Archiver
from bdfr.cloner import RedditCloner
from bdfr.configuration import Configuration
from bdfr.connector import RedditConnector
from bdfr.downloader import RedditDownloader
from bdfr.archiver import Archiver
from bdfr.cloner import RedditCloner
from bdfr import exceptions as errors
logger = logging.getLogger(__name__)
@@ -84,7 +84,9 @@ def retry_reddit_api_call(api_call_func: Callable, max_retries: int = 5, base_wa
raise Exception(f"Reddit API rate limited after {max_retries + 1} attempts: {e}")
# Wait before retrying
logger.info(f"Reddit API rate limited, waiting {current_wait_time} seconds before retry {attempt + 1}/{max_retries}")
logger.info(
f"Reddit API rate limited, waiting {current_wait_time} seconds before retry {attempt + 1}/{max_retries}"
)
time.sleep(current_wait_time)
# Increase wait time for next attempt (exponential backoff)
@@ -93,6 +95,7 @@ def retry_reddit_api_call(api_call_func: Callable, max_retries: int = 5, base_wa
class DownloadType(Enum):
"""Types of downloads supported by BDFR"""
SUBREDDIT = "subreddit"
USER = "user"
MULTIREDDIT = "multireddit"
@@ -103,6 +106,7 @@ class DownloadType(Enum):
class DownloadStatus(Enum):
"""Status of a download operation"""
QUEUED = "queued"
RUNNING = "running"
COMPLETED = "completed"
@@ -121,7 +125,7 @@ class ProgressEvent:
message: str,
progress: Optional[float] = None,
data: Optional[Dict[str, Any]] = None,
timestamp: Optional[datetime] = None
timestamp: Optional[datetime] = None,
):
self.event_type = event_type # "progress", "status", "error", "completed"
self.download_id = download_id
@@ -138,7 +142,7 @@ class ProgressEvent:
"message": self.message,
"progress": self.progress,
"data": self.data,
"timestamp": self.timestamp.isoformat()
"timestamp": self.timestamp.isoformat(),
}
@@ -177,8 +181,8 @@ class LoggingCallback(ProgressCallback):
async def on_error(self, event: ProgressEvent):
"""Log error events"""
self.logger.error(f"[{event.download_id}] ERROR: {event.message}")
if event.data.get('exception'):
self.logger.exception(f"[{event.download_id}] Exception details", exc_info=event.data['exception'])
if event.data.get("exception"):
self.logger.exception(f"[{event.download_id}] Exception details", exc_info=event.data["exception"])
async def on_completed(self, event: ProgressEvent):
"""Log completion events"""
@@ -258,7 +262,7 @@ class BDFRManager:
download_type: DownloadType,
name: str,
config: Optional[Configuration] = None,
progress_callbacks: Optional[List[ProgressCallback]] = None
progress_callbacks: Optional[List[ProgressCallback]] = None,
) -> str:
"""
Create a new download operation.
@@ -326,7 +330,7 @@ class BDFRManager:
"items_processed": 0,
"items_found": 0,
"current_item": None,
"phase": "queued"
"phase": "queued",
}
self.logger.info(f"Created download {download_id} for {download_type.value}: {name}")
@@ -356,11 +360,7 @@ class BDFRManager:
download_info["start_time"] = datetime.now()
# Create and start download thread
download_thread = threading.Thread(
target=self._run_download,
args=(download_id,),
daemon=True
)
download_thread = threading.Thread(target=self._run_download, args=(download_id,), daemon=True)
self._download_threads[download_id] = download_thread
download_thread.start()
@@ -375,8 +375,12 @@ class BDFRManager:
downloader = None
try:
self.logger.info(f"[DEBUG] Starting download {download_id} for {download_info['type']}: {download_info['name']}")
self.logger.info(f"[DEBUG] Config: subreddit={getattr(config, 'subreddit', None)}, user={getattr(config, 'user', None)}, limit={getattr(config, 'limit', None)}")
self.logger.info(
f"[DEBUG] Starting download {download_id} for {download_info['type']}: {download_info['name']}"
)
self.logger.info(
f"[DEBUG] Config: subreddit={getattr(config, 'subreddit', None)}, user={getattr(config, 'user', None)}, limit={getattr(config, 'limit', None)}"
)
# Create progress event for start
start_event = ProgressEvent(
@@ -384,11 +388,13 @@ class BDFRManager:
download_id,
f"Starting {download_info['type']} download: {download_info['name']}",
0.0,
{"phase": "starting"}
{"phase": "starting"},
)
# Notify callbacks
self.logger.info(f"[BDFR-API] Notifying {len(callbacks)} callbacks of progress event for download {download_id}")
self.logger.info(
f"[BDFR-API] Notifying {len(callbacks)} callbacks of progress event for download {download_id}"
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", start_event))
# Set up authentication token in config if provided
@@ -411,7 +417,9 @@ class BDFRManager:
log_path.parent.mkdir(parents=True, exist_ok=True)
# Determine which BDFR class to use based on configuration
self.logger.info(f"[DEBUG] Determining downloader type - submitted={getattr(config, 'submitted', False)}, upvoted={getattr(config, 'upvoted', False)}, saved={getattr(config, 'saved', False)}, format={getattr(config, 'format', 'N/A')}")
self.logger.info(
f"[DEBUG] Determining downloader type - submitted={getattr(config, 'submitted', False)}, upvoted={getattr(config, 'upvoted', False)}, saved={getattr(config, 'saved', False)}, format={getattr(config, 'format', 'N/A')}"
)
try:
if config.submitted or config.upvoted or config.saved:
@@ -420,7 +428,7 @@ class BDFRManager:
downloader = RedditDownloader(config, [])
download_info["operation_type"] = "user_download"
self.logger.info(f"[DEBUG] Using RedditDownloader for user data")
elif hasattr(config, 'format') and config.format in ["json", "xml", "yaml"]:
elif hasattr(config, "format") and config.format in ["json", "xml", "yaml"]:
# Archive operation
self.logger.info(f"[DEBUG] Creating Archiver for archive operation")
downloader = Archiver(config, [])
@@ -447,7 +455,9 @@ class BDFRManager:
# Mark as completed only if not already failed or cancelled
if download_info.get("status") not in [DownloadStatus.FAILED.value, DownloadStatus.CANCELLED.value]:
self.logger.info(f"[DEBUG] Marking download {download_id} as completed")
self.logger.info(f"[DEBUG] Final stats - items_processed: {download_info['items_processed']}, items_found: {download_info['items_found']}")
self.logger.info(
f"[DEBUG] Final stats - items_processed: {download_info['items_processed']}, items_found: {download_info['items_found']}"
)
download_info["status"] = DownloadStatus.COMPLETED.value
download_info["progress"] = 100.0
download_info["end_time"] = datetime.now()
@@ -460,18 +470,21 @@ class BDFRManager:
{
"items_processed": download_info["items_processed"],
"items_found": download_info["items_found"],
"operation_type": download_info.get("operation_type")
}
"operation_type": download_info.get("operation_type"),
},
)
self.logger.info(f"[DEBUG] Sending completion callback for {download_id}")
self.logger.info(f"[BDFR-API] Sending completion event for download {download_id}")
asyncio.run(self._notify_callbacks(callbacks, "on_completed", completed_event))
else:
self.logger.info(f"[DEBUG] Skipping completion for {download_id} due to status={download_info.get('status')}")
self.logger.info(
f"[DEBUG] Skipping completion for {download_id} due to status={download_info.get('status')}"
)
except Exception as e:
import traceback
# Get full stack trace for debugging
stack_trace = traceback.format_exc()
@@ -479,6 +492,7 @@ class BDFRManager:
is_rate_limited = False
try:
import prawcore
TooManyRequests = getattr(prawcore.exceptions, "TooManyRequests", None)
if TooManyRequests is not None and isinstance(e, TooManyRequests):
is_rate_limited = True
@@ -503,7 +517,11 @@ class BDFRManager:
download_id,
f"Download failed: {str(e)}",
None,
{"exception": str(e), "stack_trace": stack_trace, "phase": ("rate_limited" if is_rate_limited else "failed")}
{
"exception": str(e),
"stack_trace": stack_trace,
"phase": ("rate_limited" if is_rate_limited else "failed"),
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
@@ -519,17 +537,19 @@ class BDFRManager:
# Close any logging handlers that might be holding file locks
logger = logging.getLogger()
for handler in logger.handlers[:]:
if hasattr(handler, 'close') and hasattr(handler, 'baseFilename'):
if hasattr(handler, "close") and hasattr(handler, "baseFilename"):
# Check if this handler is for our download's log file
if hasattr(config, 'log') and config.log:
handler_path = getattr(handler, 'baseFilename', '')
if hasattr(config, "log") and config.log:
handler_path = getattr(handler, "baseFilename", "")
if handler_path == config.log or handler_path.startswith(str(self.download_directory)):
try:
handler.close()
logger.removeHandler(handler)
self.logger.info(f"[DEBUG] Closed logging handler for {handler_path}")
except Exception as cleanup_error:
self.logger.warning(f"[DEBUG] Error closing handler {handler_path}: {cleanup_error}")
self.logger.warning(
f"[DEBUG] Error closing handler {handler_path}: {cleanup_error}"
)
except Exception as cleanup_error:
self.logger.warning(f"[DEBUG] Error during logging cleanup: {cleanup_error}")
@@ -549,17 +569,13 @@ class BDFRManager:
# Update phase to fetching
download_info["phase"] = "fetching_submissions"
progress_event = ProgressEvent(
"progress",
download_id,
"Fetching submissions...",
10.0,
{"phase": "fetching_submissions"}
"progress", download_id, "Fetching submissions...", 10.0, {"phase": "fetching_submissions"}
)
self.logger.info(f"[BDFR-API] Sending progress update for download {download_id}")
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
# For RedditDownloader, we need to intercept the download process
if hasattr(downloader, 'reddit_lists') and hasattr(downloader, '_download_submission'):
if hasattr(downloader, "reddit_lists") and hasattr(downloader, "_download_submission"):
processed_submissions = 0
self.logger.info(f"[DEBUG] RedditDownloader has reddit_lists and _download_submission methods")
@@ -567,7 +583,7 @@ class BDFRManager:
# Get the limit from config - this is our target/goal for items_found
# items_found represents "how many items are available to download (up to limit)"
# items_processed represents "how many we've actually processed so far"
limit = getattr(config, 'limit', 100)
limit = getattr(config, "limit", 100)
download_info["items_found"] = limit
self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}")
@@ -590,7 +606,9 @@ class BDFRManager:
return
try:
self.logger.info(f"[DEBUG] Processing submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}")
self.logger.info(
f"[DEBUG] Processing submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}"
)
# Update current item being processed
download_info["current_item"] = submission.id
@@ -614,8 +632,8 @@ class BDFRManager:
"current_item_type": "submission",
"current_subreddit": submission.subreddit.display_name,
"items_processed": processed_submissions,
"items_found": download_info["items_found"] # Keep showing the limit as target
}
"items_found": download_info["items_found"], # Keep showing the limit as target
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
@@ -628,8 +646,12 @@ class BDFRManager:
# Check if this is a rate limiting error that should be retried
error_msg = str(e)
if "429" in error_msg:
self.logger.warning(f"[DEBUG] Rate limited while processing submission {submission.id}: {e}")
self.logger.warning(f"[DEBUG] Will continue with next submission instead of failing entire download")
self.logger.warning(
f"[DEBUG] Rate limited while processing submission {submission.id}: {e}"
)
self.logger.warning(
f"[DEBUG] Will continue with next submission instead of failing entire download"
)
continue
else:
# Not a rate limiting error, re-raise
@@ -640,6 +662,7 @@ class BDFRManager:
except Exception as e:
import traceback
error_msg = str(e)
submission_id = submission.id if submission is not None else "unknown"
@@ -647,7 +670,9 @@ class BDFRManager:
stack_trace = traceback.format_exc()
if "429" in error_msg:
self.logger.error(f"[DEBUG] Rate limited while processing submission {submission_id}: {e}")
self.logger.error(
f"[DEBUG] Rate limited while processing submission {submission_id}: {e}"
)
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
# Mark download as failed due to rate limiting
download_info["status"] = DownloadStatus.FAILED.value
@@ -659,7 +684,7 @@ class BDFRManager:
download_id,
f"Rate limited by Reddit API: {e}",
None,
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"}
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"},
)
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
return
@@ -675,9 +700,11 @@ class BDFRManager:
if actual_count < limit:
download_info["items_found"] = actual_count
self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}")
self.logger.info(f"[DEBUG] Final progress update - processed {processed_submissions} out of {download_info['items_found']} found")
self.logger.info(
f"[DEBUG] Final progress update - processed {processed_submissions} out of {download_info['items_found']} found"
)
progress_event = ProgressEvent(
"progress",
download_id,
@@ -686,26 +713,26 @@ class BDFRManager:
{
"phase": "completed",
"items_processed": processed_submissions,
"items_found": download_info["items_found"]
}
"items_found": download_info["items_found"],
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
elif hasattr(downloader, 'reddit_lists') and hasattr(downloader, 'write_entry'):
elif hasattr(downloader, "reddit_lists") and hasattr(downloader, "write_entry"):
# For Archiver, we need to intercept the archiving process
processed_items = 0
self.logger.info(f"[DEBUG] Archiver detected with reddit_lists and write_entry methods")
# Get the limit from config
limit = getattr(config, 'limit', 100)
limit = getattr(config, "limit", 100)
download_info["items_found"] = limit
self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}")
# Process items with progress tracking
self.logger.info(f"[DEBUG] Starting archiving loop...")
actual_count = 0
for generator in downloader.reddit_lists:
self.logger.info(f"[DEBUG] Processing generator for archiving...")
item_count = 0
@@ -714,14 +741,15 @@ class BDFRManager:
continue
item_count += 1
actual_count += 1
if download_info["status"] == DownloadStatus.CANCELLED.value:
self.logger.info(f"Download {download_id} was cancelled")
return
try:
# Determine item type
import praw.models
if isinstance(item, praw.models.Submission):
item_id = item.id
item_type = "submission"
@@ -729,23 +757,27 @@ class BDFRManager:
elif isinstance(item, praw.models.Comment):
item_id = item.id
item_type = "comment"
subreddit_name = item.subreddit.display_name if hasattr(item, 'subreddit') else "unknown"
subreddit_name = (
item.subreddit.display_name if hasattr(item, "subreddit") else "unknown"
)
else:
item_id = str(item)
item_type = "item"
subreddit_name = "unknown"
self.logger.info(f"[DEBUG] Archiving {item_type} {item_count}: {item_id} from r/{subreddit_name}")
self.logger.info(
f"[DEBUG] Archiving {item_type} {item_count}: {item_id} from r/{subreddit_name}"
)
# Update current item being processed
download_info["current_item"] = item_id
download_info["current_item_type"] = item_type
download_info["current_subreddit"] = subreddit_name
# Send progress update for this item
processed_items += 1
progress_percent = 20.0 + (processed_items / max(limit, 1)) * 70.0
progress_event = ProgressEvent(
"progress",
download_id,
@@ -757,11 +789,11 @@ class BDFRManager:
"current_item_type": item_type,
"current_subreddit": subreddit_name,
"items_processed": processed_items,
"items_found": download_info["items_found"]
}
"items_found": download_info["items_found"],
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
# Archive this item with retry mechanism for rate limiting
self.logger.info(f"[DEBUG] Calling write_entry for {item_id}")
try:
@@ -772,34 +804,37 @@ class BDFRManager:
error_msg = str(e)
if "429" in error_msg:
self.logger.warning(f"[DEBUG] Rate limited while archiving item {item_id}: {e}")
self.logger.warning(f"[DEBUG] Will continue with next item instead of failing entire download")
self.logger.warning(
f"[DEBUG] Will continue with next item instead of failing entire download"
)
continue
else:
# Not a rate limiting error, re-raise
raise
# Update processed count
download_info["items_processed"] = processed_items
except Exception as e:
import traceback
error_msg = str(e)
item_id = item_id if 'item_id' in locals() else "unknown"
item_id = item_id if "item_id" in locals() else "unknown"
stack_trace = traceback.format_exc()
if "429" in error_msg:
self.logger.error(f"[DEBUG] Rate limited while archiving item {item_id}: {e}")
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
download_info["status"] = DownloadStatus.FAILED.value
download_info["error"] = f"Rate limited by Reddit API: {e}"
download_info["end_time"] = datetime.now()
error_event = ProgressEvent(
"error",
download_id,
f"Rate limited by Reddit API: {e}",
None,
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"}
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"},
)
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
return
@@ -807,16 +842,18 @@ class BDFRManager:
self.logger.error(f"[DEBUG] Error archiving item {item_id}: {e}")
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
continue
self.logger.info(f"[DEBUG] Generator archived {item_count} items")
# Final progress update
if actual_count < limit:
download_info["items_found"] = actual_count
self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}")
self.logger.info(f"[DEBUG] Final progress update - archived {processed_items} out of {download_info['items_found']} found")
self.logger.info(
f"[DEBUG] Final progress update - archived {processed_items} out of {download_info['items_found']} found"
)
progress_event = ProgressEvent(
"progress",
download_id,
@@ -825,26 +862,32 @@ class BDFRManager:
{
"phase": "completed",
"items_processed": processed_items,
"items_found": download_info["items_found"]
}
"items_found": download_info["items_found"],
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
elif hasattr(downloader, 'reddit_lists') and hasattr(downloader, '_download_submission') and hasattr(downloader, 'write_entry'):
elif (
hasattr(downloader, "reddit_lists")
and hasattr(downloader, "_download_submission")
and hasattr(downloader, "write_entry")
):
# For RedditCloner (which has both download and archive capabilities)
processed_items = 0
self.logger.info(f"[DEBUG] RedditCloner detected with reddit_lists, _download_submission, and write_entry methods")
self.logger.info(
f"[DEBUG] RedditCloner detected with reddit_lists, _download_submission, and write_entry methods"
)
# Get the limit from config
limit = getattr(config, 'limit', 100)
limit = getattr(config, "limit", 100)
download_info["items_found"] = limit
self.logger.info(f"[DEBUG] Set items_found (target) to limit: {limit}")
# Process submissions with progress tracking (both download and archive)
self.logger.info(f"[DEBUG] Starting cloning loop...")
actual_count = 0
for generator in downloader.reddit_lists:
self.logger.info(f"[DEBUG] Processing generator for cloning...")
submission_count = 0
@@ -853,23 +896,25 @@ class BDFRManager:
continue
submission_count += 1
actual_count += 1
if download_info["status"] == DownloadStatus.CANCELLED.value:
self.logger.info(f"Download {download_id} was cancelled")
return
try:
self.logger.info(f"[DEBUG] Cloning submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}")
self.logger.info(
f"[DEBUG] Cloning submission {submission_count}: {submission.id} from r/{submission.subreddit.display_name}"
)
# Update current item being processed
download_info["current_item"] = submission.id
download_info["current_item_type"] = "submission"
download_info["current_subreddit"] = submission.subreddit.display_name
# Send progress update for this submission
processed_items += 1
progress_percent = 20.0 + (processed_items / max(limit, 1)) * 70.0
progress_event = ProgressEvent(
"progress",
download_id,
@@ -881,11 +926,11 @@ class BDFRManager:
"current_item_type": "submission",
"current_subreddit": submission.subreddit.display_name,
"items_processed": processed_items,
"items_found": download_info["items_found"]
}
"items_found": download_info["items_found"],
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
# Clone this submission (download + archive) with retry mechanism for rate limiting
self.logger.info(f"[DEBUG] Calling _download_submission for {submission.id}")
try:
@@ -897,36 +942,43 @@ class BDFRManager:
# Check if this is a rate limiting error that should be retried
error_msg = str(e)
if "429" in error_msg:
self.logger.warning(f"[DEBUG] Rate limited while cloning submission {submission.id}: {e}")
self.logger.warning(f"[DEBUG] Will continue with next submission instead of failing entire download")
self.logger.warning(
f"[DEBUG] Rate limited while cloning submission {submission.id}: {e}"
)
self.logger.warning(
f"[DEBUG] Will continue with next submission instead of failing entire download"
)
continue
else:
# Not a rate limiting error, re-raise
raise
# Update processed count
download_info["items_processed"] = processed_items
except Exception as e:
import traceback
error_msg = str(e)
submission_id = submission.id if submission is not None else "unknown"
stack_trace = traceback.format_exc()
if "429" in error_msg:
self.logger.error(f"[DEBUG] Rate limited while cloning submission {submission_id}: {e}")
self.logger.error(
f"[DEBUG] Rate limited while cloning submission {submission_id}: {e}"
)
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
self.logger.info(f"[DEBUG] Error message contains '429': {error_msg}")
download_info["status"] = DownloadStatus.FAILED.value
download_info["error"] = f"Rate limited by Reddit API: {e}"
download_info["end_time"] = datetime.now()
error_event = ProgressEvent(
"error",
download_id,
f"Rate limited by Reddit API: {e}",
None,
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"}
{"exception": str(e), "stack_trace": stack_trace, "phase": "rate_limited"},
)
asyncio.run(self._notify_callbacks(callbacks, "on_error", error_event))
return
@@ -934,16 +986,18 @@ class BDFRManager:
self.logger.error(f"[DEBUG] Error cloning submission {submission_id}: {e}")
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
continue
self.logger.info(f"[DEBUG] Generator cloned {submission_count} submissions")
# Final progress update
if actual_count < limit:
download_info["items_found"] = actual_count
self.logger.info(f"[DEBUG] Adjusted items_found from {limit} to actual count {actual_count}")
self.logger.info(f"[DEBUG] Final progress update - cloned {processed_items} out of {download_info['items_found']} found")
self.logger.info(
f"[DEBUG] Final progress update - cloned {processed_items} out of {download_info['items_found']} found"
)
progress_event = ProgressEvent(
"progress",
download_id,
@@ -952,11 +1006,11 @@ class BDFRManager:
{
"phase": "completed",
"items_processed": processed_items,
"items_found": download_info["items_found"]
}
"items_found": download_info["items_found"],
},
)
asyncio.run(self._notify_callbacks(callbacks, "on_progress", progress_event))
else:
# For other downloaders, just run normally
self.logger.info(f"[DEBUG] Using fallback downloader (no specific handler)")
@@ -964,6 +1018,7 @@ class BDFRManager:
except Exception as e:
import traceback
stack_trace = traceback.format_exc()
self.logger.error(f"Error in progress download for {download_id}: {e}")
self.logger.error(f"[DEBUG] Stack trace: {stack_trace}")
@@ -971,7 +1026,7 @@ class BDFRManager:
raise
# Set up authentication if token provided
if hasattr(config, 'auth_token') and config.auth_token:
if hasattr(config, "auth_token") and config.auth_token:
self.logger.info(f"[DEBUG] Setting up authenticated Reddit instance")
# Monkey patch the create_reddit_instance method to use our token
original_create_reddit_instance = downloader.create_reddit_instance
@@ -985,11 +1040,12 @@ class BDFRManager:
# Override the reddit_instance with authenticated version
import praw
downloader.reddit_instance = praw.Reddit(
client_id=downloader.cfg_parser.get("DEFAULT", "client_id"),
client_secret=downloader.cfg_parser.get("DEFAULT", "client_secret"),
user_agent=getattr(downloader, 'user_agent', 'BDFR-Web-Interface/1.0'),
token=auth_token
user_agent=getattr(downloader, "user_agent", "BDFR-Web-Interface/1.0"),
token=auth_token,
)
downloader.authenticated = True
self.logger.info(f"[DEBUG] Created authenticated Reddit instance successfully")
@@ -1005,10 +1061,14 @@ class BDFRManager:
async def _notify_callbacks(self, callbacks: List[ProgressCallback], method_name: str, event: ProgressEvent):
"""Notify all callbacks of an event"""
self.logger.info(f"[BDFR-API] Notifying {len(callbacks)} callbacks of {method_name} event for download {event.download_id}")
self.logger.info(
f"[BDFR-API] Notifying {len(callbacks)} callbacks of {method_name} event for download {event.download_id}"
)
for callback in callbacks:
try:
logger.info(f"[BDFR-API] Invoking {method_name} on callback {callback.__class__.__name__} for download {event.download_id}")
logger.info(
f"[BDFR-API] Invoking {method_name} on callback {callback.__class__.__name__} for download {event.download_id}"
)
method = getattr(callback, method_name)
if asyncio.iscoroutinefunction(method):
await method(event)
@@ -1080,10 +1140,7 @@ class BDFRManager:
Returns:
Dictionary of download information keyed by download ID
"""
return {
download_id: self.get_download_status(download_id)
for download_id in self._active_downloads
}
return {download_id: self.get_download_status(download_id) for download_id in self._active_downloads}
def cleanup_completed(self, max_age_seconds: int = 3600) -> int:
"""
@@ -1099,7 +1156,11 @@ class BDFRManager:
to_remove = []
for download_id, download_info in self._active_downloads.items():
if download_info["status"] in [DownloadStatus.COMPLETED.value, DownloadStatus.FAILED.value, DownloadStatus.CANCELLED.value]:
if download_info["status"] in [
DownloadStatus.COMPLETED.value,
DownloadStatus.FAILED.value,
DownloadStatus.CANCELLED.value,
]:
end_time = download_info.get("end_time")
if end_time and (current_time - end_time).total_seconds() > max_age_seconds:
to_remove.append(download_id)
@@ -1124,7 +1185,7 @@ class BDFRManager:
time_filter: str = "all",
no_dupes: bool = False,
simple_check: bool = False,
progress_callbacks: Optional[List[ProgressCallback]] = None
progress_callbacks: Optional[List[ProgressCallback]] = None,
) -> str:
"""
Download content from a subreddit.
@@ -1168,12 +1229,7 @@ class BDFRManager:
config.no_dupes = no_dupes
config.simple_check = simple_check
download_id = self.create_download(
DownloadType.SUBREDDIT,
subreddit_name,
config,
progress_callbacks
)
download_id = self.create_download(DownloadType.SUBREDDIT, subreddit_name, config, progress_callbacks)
self.start_download(download_id)
return download_id
@@ -1187,7 +1243,7 @@ class BDFRManager:
saved: bool = False,
no_dupes: bool = False,
simple_check: bool = False,
progress_callbacks: Optional[List[ProgressCallback]] = None
progress_callbacks: Optional[List[ProgressCallback]] = None,
) -> str:
"""
Download content from a user.
@@ -1234,7 +1290,7 @@ class BDFRManager:
config.saved = saved
config.no_dupes = no_dupes
config.simple_check = simple_check
# Set authentication if token is available for this manager instance
if self.auth_token:
config.authenticate = True
@@ -1244,12 +1300,7 @@ class BDFRManager:
config.authenticate = False
logger.info(f"[DEBUG] No authentication token available for user download")
download_id = self.create_download(
DownloadType.USER,
username,
config,
progress_callbacks
)
download_id = self.create_download(DownloadType.USER, username, config, progress_callbacks)
self.start_download(download_id)
return download_id
@@ -1260,7 +1311,7 @@ class BDFRManager:
format_type: str = "json",
limit: Optional[int] = None,
simple_check: bool = False,
progress_callbacks: Optional[List[ProgressCallback]] = None
progress_callbacks: Optional[List[ProgressCallback]] = None,
) -> str:
"""
Archive subreddit data (metadata only, no downloads).
@@ -1298,12 +1349,7 @@ class BDFRManager:
config.limit = limit
config.format = format_type
download_id = self.create_download(
DownloadType.ARCHIVE,
subreddit_name,
config,
progress_callbacks
)
download_id = self.create_download(DownloadType.ARCHIVE, subreddit_name, config, progress_callbacks)
self.start_download(download_id)
return download_id
@@ -1315,7 +1361,7 @@ class BDFRManager:
format_type: str = "json",
no_dupes: bool = False,
simple_check: bool = False,
progress_callbacks: Optional[List[ProgressCallback]] = None
progress_callbacks: Optional[List[ProgressCallback]] = None,
) -> str:
"""
Clone subreddit (both download and archive).
@@ -1355,12 +1401,7 @@ class BDFRManager:
config.format = format_type
config.no_dupes = no_dupes
download_id = self.create_download(
DownloadType.CLONE,
subreddit_name,
config,
progress_callbacks
)
download_id = self.create_download(DownloadType.CLONE, subreddit_name, config, progress_callbacks)
self.start_download(download_id)
return download_id
@@ -1394,4 +1435,4 @@ def set_bdfr_manager(manager: BDFRManager):
manager: BDFRManager instance to use as default
"""
global _default_manager
_default_manager = manager
_default_manager = manager