ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reformatting
formatting_check / formatting_check (push) Failing after 3s
Details
Python Test / test (.ps1, windows-latest, 3.9) (push) Has been cancelled
Details
Python Test / test (.sh, macos-latest, 3.9) (push) Has been cancelled
Details
Python Test / test (.sh, ubuntu-latest, 3.9) (push) Has been cancelled
Details

Browse Source
This commit is contained in:
ModerateWinGuy
2026-07-14 21:32:55 +12:00
parent 3d0658b483
commit 0157f462cc
17 changed files with 439 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bdfr/downloader.py
+39 -39
View File
@@ -50,11 +50,13 @@ class RedditDownloader(RedditConnector):
hash_data = self._load_hash_list()
# Handle both old and new hash file formats
if isinstance(hash_data, dict) and 'files' in hash_data:
if isinstance(hash_data, dict) and "files" in hash_data:
# New format with enhanced structure
self.master_hash_list = {k: v['path'] for k, v in hash_data['files'].items()}
self.url_list = hash_data.get('urls', {})
logger.info(f"Loaded {len(self.master_hash_list)} hashes and {len(self.url_list)} URLs from enhanced hash file")
self.master_hash_list = {k: v["path"] for k, v in hash_data["files"].items()}
self.url_list = hash_data.get("urls", {})
logger.info(
f"Loaded {len(self.master_hash_list)} hashes and {len(self.url_list)} URLs from enhanced hash file"
)
else:
# Old format - just hashes
self.master_hash_list = hash_data
@@ -70,8 +72,10 @@ class RedditDownloader(RedditConnector):
if hash_value not in existing_hashes:
self.master_hash_list[hash_value] = file_path
logger.info(f"Loaded {len(self.master_hash_list)} total hashes "
f"({len(existing_hashes)} from file, {len(all_files_hashes) - len(existing_hashes)} new)")
logger.info(
f"Loaded {len(self.master_hash_list)} total hashes "
f"({len(existing_hashes)} from file, {len(all_files_hashes) - len(existing_hashes)} new)"
)
def download(self):
for generator in self.reddit_lists:
@@ -164,7 +168,7 @@ class RedditDownloader(RedditConnector):
self.master_hash_list[existing_file_hash] = destination
# Store URL mapping for simple-check functionality if URL is available
if hasattr(res, 'url') and self.args.simple_check:
if hasattr(res, "url") and self.args.simple_check:
self.url_list[res.url] = existing_file_hash
logger.debug(f"Added hash for existing file: {existing_file_hash}")
@@ -190,7 +194,7 @@ class RedditDownloader(RedditConnector):
resource_hash = res.hash.hexdigest()
# Simple-check: URL-based duplicate detection (fast path)
if self.args.simple_check and hasattr(res, 'url') and res.url in self.url_list:
if self.args.simple_check and hasattr(res, "url") and res.url in self.url_list:
stored_hash = self.url_list[res.url]
if stored_hash in self.master_hash_list:
logger.info(f"URL {res.url} from submission {submission.id} already downloaded (simple-check)")
@@ -233,7 +237,7 @@ class RedditDownloader(RedditConnector):
self.master_hash_list[resource_hash] = destination
# Store URL mapping for simple-check functionality
if hasattr(res, 'url') and self.args.simple_check:
if hasattr(res, "url") and self.args.simple_check:
self.url_list[res.url] = resource_hash
logger.debug(f"Hash added to master list: {resource_hash}")
@@ -252,7 +256,7 @@ class RedditDownloader(RedditConnector):
@staticmethod
def scan_existing_files(directory: Path) -> dict[str, Path]:
files = []
for (dirpath, _dirnames, filenames) in os.walk(directory):
for dirpath, _dirnames, filenames in os.walk(directory):
files.extend([Path(dirpath, file) for file in filenames])
logger.info(f"Calculating hashes for {len(files)} files")
@@ -270,14 +274,14 @@ class RedditDownloader(RedditConnector):
def _load_hash_list(self) -> dict[str, Path]:
"""Load existing hash list from .bdfr_hashes.json in download directory."""
logger.debug(f"Loading hash list from directory: {self.download_directory}")
hash_file_path = self.download_directory / '.bdfr_hashes.json'
hash_file_path = self.download_directory / ".bdfr_hashes.json"
if not hash_file_path.exists():
logger.debug(f"No existing hash file found at {hash_file_path}")
return {}
try:
with open(hash_file_path, 'r', encoding='utf-8') as f:
with open(hash_file_path, "r", encoding="utf-8") as f:
hash_data = json.load(f)
if not isinstance(hash_data, dict):
@@ -285,16 +289,16 @@ class RedditDownloader(RedditConnector):
return {}
# Handle new enhanced format
if 'files' in hash_data and isinstance(hash_data['files'], dict):
if "files" in hash_data and isinstance(hash_data["files"], dict):
# New format with enhanced structure
files_data = hash_data['files']
files_data = hash_data["files"]
loaded_hashes = {}
urls_data = hash_data.get('urls', {})
urls_data = hash_data.get("urls", {})
for hash_value, file_info in files_data.items():
if isinstance(file_info, dict) and 'path' in file_info:
if isinstance(file_info, dict) and "path" in file_info:
# New format: {"hash": {"path": "relative/path", "url": "http://..."}}
relative_path = file_info['path']
relative_path = file_info["path"]
absolute_path = self.download_directory / relative_path
if absolute_path.exists():
loaded_hashes[hash_value] = absolute_path
@@ -302,8 +306,8 @@ class RedditDownloader(RedditConnector):
logger.debug(f"File {absolute_path} from hash file no longer exists")
# Load URL mapping for simple-check
if 'url' in file_info and file_info['url']:
self.url_list[file_info['url']] = hash_value
if "url" in file_info and file_info["url"]:
self.url_list[file_info["url"]] = hash_value
elif isinstance(file_info, str):
# Legacy format within new structure: {"hash": "relative/path"}
absolute_path = self.download_directory / file_info
@@ -337,30 +341,30 @@ class RedditDownloader(RedditConnector):
def _save_hash_list(self) -> None:
"""Save current hash list to .bdfr_hashes.json in download directory using atomic write."""
hash_file_path = self.download_directory / '.bdfr_hashes.json'
hash_file_path = self.download_directory / ".bdfr_hashes.json"
# Build enhanced data structure for new format
if self.args.simple_check:
# New enhanced format with URLs and metadata
hash_data = {
'files': {},
'urls': self.url_list.copy(),
'metadata': {
'version': '2.0',
'created_with': 'simple_check' if self.args.simple_check else 'standard',
'url_count': len(self.url_list),
'hash_count': len(self.master_hash_list)
}
"files": {},
"urls": self.url_list.copy(),
"metadata": {
"version": "2.0",
"created_with": "simple_check" if self.args.simple_check else "standard",
"url_count": len(self.url_list),
"hash_count": len(self.master_hash_list),
},
}
# Convert absolute paths to relative paths for portability
for hash_value, absolute_path in self.master_hash_list.items():
try:
relative_path = absolute_path.relative_to(self.download_directory)
hash_data['files'][hash_value] = {
'path': str(relative_path),
'url': next((url for url, h in self.url_list.items() if h == hash_value), None),
'check_method': 'hash'
hash_data["files"][hash_value] = {
"path": str(relative_path),
"url": next((url for url, h in self.url_list.items() if h == hash_value), None),
"check_method": "hash",
}
except ValueError:
# File is not relative to download directory, skip it
@@ -381,17 +385,13 @@ class RedditDownloader(RedditConnector):
# Atomic write: write to temporary file first, then rename
try:
with tempfile.NamedTemporaryFile(
mode='w',
dir=self.download_directory,
suffix='.tmp',
delete=False,
encoding='utf-8'
mode="w", dir=self.download_directory, suffix=".tmp", delete=False, encoding="utf-8"
) as temp_file:
json.dump(hash_data, temp_file, indent=2)
temp_file_path = temp_file.name
# Atomic rename
if os.name == 'nt': # Windows
if os.name == "nt": # Windows
# On Windows, we need to remove the target file first if it exists
if hash_file_path.exists():
hash_file_path.unlink()
@@ -407,7 +407,7 @@ class RedditDownloader(RedditConnector):
logger.error(f"Unexpected error saving hash file {hash_file_path}: {e}")
# Clean up temp file if it still exists
try:
if 'temp_file_path' in locals():
if "temp_file_path" in locals():
os.unlink(temp_file_path)
except (OSError, IOError):
pass