ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed misc bugs

Browse Source
This commit is contained in:
ModerateWinGuy
2025-10-15 13:31:18 +13:00
parent 9f5a25fcf5
commit 7580dc3f94
13 changed files with 609 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bdfr/api.py
+1 -1
View File
@@ -112,7 +112,7 @@ class LoggingCallback(ProgressCallback):
async def on_progress(self, event: ProgressEvent):
"""Log progress events"""
if event.progress is not None:
self.logger.info(f"[{event.download_id}] {event.message} ({event.progress}%)")
self.logger.info(f"[{event.download_id}] {event.message} ({int(round(event.progress))}%)")
else:
self.logger.info(f"[{event.download_id}] {event.message}")
bdfr/connector.py
+26 -4
View File
@@ -195,16 +195,38 @@ class RedditConnector(metaclass=ABCMeta):
Path(self.config_directory, "config.cfg"),
Path(self.config_directory, "default_config.cfg"),
]
logger.debug(f"Config directory is: {self.config_directory}")
logger.debug(f"Checking possible config paths: {[str(p) for p in possible_paths]}")
self.config_location = None
for path in possible_paths:
if path.resolve().expanduser().exists():
resolved_path = path.resolve().expanduser()
logger.debug(f"Checking if {resolved_path} exists: {resolved_path.exists()}")
if resolved_path.exists():
self.config_location = path
logger.debug(f"Loading configuration from {path}")
break
if not self.config_location:
with importlib.resources.path("bdfr", "default_config.cfg") as path:
self.config_location = path
shutil.copy(self.config_location, Path(self.config_directory, "default_config.cfg"))
# Try to use a fallback location that avoids importlib.resources context manager issues
# when running as non-root user in Docker
fallback_config = Path("/tmp/bdfr_default_config.cfg")
if fallback_config.exists():
logger.debug("Using fallback config from /tmp/bdfr_default_config.cfg")
shutil.copy(fallback_config, Path(self.config_directory, "default_config.cfg"))
self.config_location = Path(self.config_directory, "default_config.cfg")
else:
# Fall back to importlib.resources if no fallback is available
try:
with importlib.resources.path("bdfr", "default_config.cfg") as path:
self.config_location = path
shutil.copy(self.config_location, Path(self.config_directory, "default_config.cfg"))
except (PermissionError, OSError) as e:
logger.error(f"Failed to access default config via importlib.resources: {e}")
# Last resort: try to read from package directory directly
package_config = Path("/usr/local/lib/python3.11/site-packages/bdfr/default_config.cfg")
if package_config.exists():
logger.debug("Using package config directly")
shutil.copy(package_config, Path(self.config_directory, "default_config.cfg"))
self.config_location = Path(self.config_directory, "default_config.cfg")
if not self.config_location:
raise errors.BulkDownloaderException("Could not find a configuration file to load")
self.cfg_parser.read(self.config_location)
bdfr/downloader.py
+5 -2
View File
@@ -153,7 +153,7 @@ class RedditDownloader(RedditConnector):
if destination.exists():
# Check if we already have this file's hash
if destination in self.master_hash_list.values():
logger.debug(f"File {destination} from submission {submission.id} already exists, continuing")
logger.info(f"File {destination.name} from submission {submission.id} already exists")
continue
else:
# File exists but not in our hash list - calculate its hash
@@ -166,6 +166,7 @@ class RedditDownloader(RedditConnector):
self.url_list[res.url] = existing_file_hash
logger.debug(f"Added hash for existing file: {existing_file_hash}")
logger.info(f"File {destination.name} from submission {submission.id} already exists")
files_processed += 1
if self.args.no_dupes:
self._save_hash_list()
@@ -185,7 +186,6 @@ class RedditDownloader(RedditConnector):
)
return
resource_hash = res.hash.hexdigest()
destination.parent.mkdir(parents=True, exist_ok=True)
# Simple-check: URL-based duplicate detection (fast path)
if self.args.simple_check and hasattr(res, 'url') and res.url in self.url_list:
@@ -213,6 +213,9 @@ class RedditDownloader(RedditConnector):
if self.args.no_dupes:
self._save_hash_list()
return
# Only create folder if we're actually going to write the file (not a duplicate)
destination.parent.mkdir(parents=True, exist_ok=True)
try:
with destination.open("wb") as file:
file.write(res.content)
bdfr/examples/api_usage.py
+6 -6
View File
@@ -46,7 +46,7 @@ class WebSocketCallback(ProgressCallback):
"""Send progress update to WebSocket"""
print(f"📊 [{self.websocket_id}] Progress: {event.message}")
if event.progress is not None:
print(f" Progress: {event.progress:.1f}%")
print(f" Progress: {int(round(event.progress))}%")
if event.data:
print(f" Data: {event.data}")
@@ -109,7 +109,7 @@ async def example_basic_usage():
print("❌ Download not found!")
break
print(f"📊 Status: {status['status']} | Progress: {status['progress']:.1f}%")
print(f"📊 Status: {status['status']} | Progress: {int(round(status['progress']))}%")
if status['status'] in ['completed', 'failed', 'cancelled']:
print(f"🏁 Download finished with status: {status['status']}")
@@ -162,7 +162,7 @@ async def example_advanced_usage():
download_ids.remove(download_id)
continue
print(f"📊 {download_id}: {status['status']} ({status['progress']:.1f}%)")
print(f"📊 {download_id}: {status['status']} ({int(round(status['progress']))}%)")
if status['status'] in ['completed', 'failed', 'cancelled']:
print(f"🏁 Download {download_id} finished")
@@ -203,7 +203,7 @@ async def example_user_download():
print("❌ Download not found")
break
print(f"📊 Status: {status['status']} | Progress: {status['progress']:.1f}%")
print(f"📊 Status: {status['status']} | Progress: {int(round(status['progress']))}%")
if status['status'] in ['completed', 'failed']:
break
@@ -236,7 +236,7 @@ async def example_archive_operation():
print("❌ Archive not found")
break
print(f"📊 Archive status: {status['status']} | Progress: {status['progress']:.1f}%")
print(f"📊 Archive status: {status['status']} | Progress: {int(round(status['progress']))}%")
if status['status'] in ['completed', 'failed']:
print(f"🏁 Archive finished with status: {status['status']}")
@@ -325,7 +325,7 @@ async def example_web_integration():
user_downloads = await app.get_user_downloads(user_id)
print(f"User has {len(user_downloads)} active downloads:")
for download in user_downloads:
print(f" - {download['id']}: {download['status']} ({download['progress']:.1f}%)")
print(f" - {download['id']}: {download['status']} ({int(round(download['progress']))}%)")
# Cancel one download
if user_downloads:
bdfr/site_downloaders/base_downloader.py
+20 -8
View File
@@ -27,11 +27,23 @@ class BaseDownloader(ABC):
@staticmethod
def retrieve_url(url: str, cookies: dict = None, headers: dict = None) -> requests.Response:
try:
res = requests.get(url, cookies=cookies, headers=headers)
except requests.exceptions.RequestException as e:
logger.exception(e)
raise SiteDownloaderError(f"Failed to get page {url}")
if res.status_code != 200:
raise ResourceNotFound(f"Server responded with {res.status_code} to {url}")
return res
max_retries = 3
for attempt in range(1, max_retries + 1):
try:
res = requests.get(url, cookies=cookies, headers=headers, timeout=10)
if res.status_code != 200:
logger.error(f"Attempt {attempt}: Server responded with {res.status_code} to {url}")
if attempt == max_retries:
raise ResourceNotFound(f"Server responded with {res.status_code} to {url}")
else:
return res
except requests.exceptions.SSLError as ssl_err:
logger.error(f"Attempt {attempt}: SSL error for {url}: {ssl_err}")
if attempt == max_retries:
raise SiteDownloaderError(f"SSL error after {max_retries} attempts for {url}: {ssl_err}")
except requests.exceptions.RequestException as e:
logger.error(f"Attempt {attempt}: Request error for {url}: {e}")
if attempt == max_retries:
raise SiteDownloaderError(f"Failed to get page {url} after {max_retries} attempts: {e}")
# Should not reach here
raise SiteDownloaderError(f"Failed to get page {url} after {max_retries} attempts")