ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(Hashing): added more hash checking for repeat download checks to avoid overlap

Browse Source
This commit is contained in:
ModerateWinGuy
2025-10-07 17:39:57 +13:00
parent 8c293a4684
commit 15f14088be
8 changed files with 584 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@@ -235,6 +235,15 @@ The following options apply only to the `download` command. This command downloa
- `--no-dupes`
- This flag will not redownload files if they were already downloaded in the current run
- This is calculated by MD5 hash
- `--simple-check`
- **Enhanced: Works with Persistent Hash Storage**
- Enables fast URL-based duplicate detection for the `--no-dupes` functionality
- When enabled, the downloader first checks if a submission URL has been downloaded before calculating expensive file hashes
- Creates enhanced hash files (`.bdfr_hashes.json`) with URL mappings for faster subsequent runs
- Stores both hash-to-file and URL-to-hash mappings for optimal performance
- Falls back to full hash checking if URL is not found in the hash file
- Maintains backward compatibility with existing hash files
- Significantly improves performance when downloading from sources with many duplicate URLs
- `--search-existing`
- This will make the BDFR compile the hashes for every file in `directory`
- The hashes are used to remove duplicates if `--no-dupes` is supplied or make hard links if `--make-hard-links` is supplied

1
bdfr/__main__.py
View File

@@ -53,6 +53,7 @@ _downloader_options = [
click.option("--max-wait-time", type=int, default=None),
click.option("--no-dupes", is_flag=True, default=None),
click.option("--search-existing", is_flag=True, default=None),
click.option("--simple-check", is_flag=True, default=None, help="Enable fast URL-based duplicate checking (works with --no-dupes)"),
click.option("--skip", default=None, multiple=True),
click.option("--skip-domain", default=None, multiple=True),
click.option("--skip-subreddit", default=None, multiple=True),

1
bdfr/configuration.py
View File

@@ -35,6 +35,7 @@ class Configuration(Namespace):
self.multireddit: list[str] = []
self.no_dupes: bool = False
self.saved: bool = False
self.simple_check: bool = False
self.search: Optional[str] = None
self.search_existing: bool = False
self.skip: list[str] = []

1
bdfr/default_config.cfg
View File

@@ -5,3 +5,4 @@ scopes = identity, history, read, save, mysubreddits
backup_log_count = 3
max_wait_time = 120
time_format = ISO

243
bdfr/downloader.py
View File

@@ -2,8 +2,10 @@
# -*- coding: utf-8 -*-
import hashlib
import json
import logging.handlers
import os
import tempfile
import time
from collections.abc import Iterable
from datetime import datetime
@@ -39,22 +41,60 @@ def _calc_hash(existing_file: Path):
class RedditDownloader(RedditConnector):
def __init__(self, args: Configuration, logging_handlers: Iterable[logging.Handler] = ()):
super(RedditDownloader, self).__init__(args, logging_handlers)
self.master_hash_list = {}
self.url_list = {} # New: Store URL to hash mapping for simple-check
# Load existing hashes if no_dupes is enabled or search_existing is requested
if self.args.no_dupes or self.args.search_existing:
# First try to load from persistent hash file
hash_data = self._load_hash_list()
# Handle both old and new hash file formats
if isinstance(hash_data, dict) and 'files' in hash_data:
# New format with enhanced structure
self.master_hash_list = {k: v['path'] for k, v in hash_data['files'].items()}
self.url_list = hash_data.get('urls', {})
logger.info(f"Loaded {len(self.master_hash_list)} hashes and {len(self.url_list)} URLs from enhanced hash file")
else:
# Old format - just hashes
self.master_hash_list = hash_data
logger.info(f"Loaded {len(self.master_hash_list)} hashes from legacy hash file")
# If search_existing is also enabled, scan for any new files not in hash list
if self.args.search_existing:
self.master_hash_list = self.scan_existing_files(self.download_directory)
existing_hashes = set(self.master_hash_list.keys())
all_files_hashes = self.scan_existing_files(self.download_directory)
# Add any new files found by scanning
for hash_value, file_path in all_files_hashes.items():
if hash_value not in existing_hashes:
self.master_hash_list[hash_value] = file_path
logger.info(f"Loaded {len(self.master_hash_list)} total hashes "
f"({len(existing_hashes)} from file, {len(all_files_hashes) - len(existing_hashes)} new)")
def download(self):
for generator in self.reddit_lists:
last_submission_id = None
try:
for submission in generator:
last_submission_id = submission.id
try:
self._download_submission(submission)
except prawcore.PrawcoreException as e:
logger.error(f"Submission {submission.id} failed to download due to a PRAW exception: {e}")
except prawcore.PrawcoreException as e:
logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
submission_id = last_submission_id or "unknown"
logger.error(f"The submission after {submission_id} failed to download due to a PRAW exception: {e}")
logger.debug("Waiting 60 seconds to continue")
sleep(60)
# Save hash list after completion if no_dupes is enabled
# Always save if no_dupes is enabled, even if hash list is empty
# This creates the hash file for future runs
if self.args.no_dupes:
self._save_hash_list()
def _download_submission(self, submission: praw.models.Submission):
if submission.id in self.excluded_submission_ids:
logger.debug(f"Object {submission.id} in exclusion list, skipping")
@@ -108,15 +148,36 @@ class RedditDownloader(RedditConnector):
except errors.SiteDownloaderError as e:
logger.error(f"Site {downloader_class.__name__} failed to download submission {submission.id}: {e}")
return
files_processed = 0
for destination, res in self.file_name_formatter.format_resource_paths(content, self.download_directory):
if destination.exists():
# Check if we already have this file's hash
if destination in self.master_hash_list.values():
logger.debug(f"File {destination} from submission {submission.id} already exists, continuing")
continue
else:
# File exists but not in our hash list - calculate its hash
try:
existing_file_hash = _calc_hash(destination)[1]
self.master_hash_list[existing_file_hash] = destination
# Store URL mapping for simple-check functionality if URL is available
if hasattr(res, 'url') and self.args.simple_check:
self.url_list[res.url] = existing_file_hash
logger.debug(f"Added hash for existing file: {existing_file_hash}")
files_processed += 1
if self.args.no_dupes:
self._save_hash_list()
except Exception as e:
logger.warning(f"Failed to calculate hash for existing file {destination}: {e}")
continue
elif not self.download_filter.check_resource(res):
logger.debug(f"Download filter removed {submission.id} file with URL {submission.url}")
continue
try:
res.download({"max_wait_time": self.args.max_wait_time})
print(f"DEBUG: Successfully downloaded resource {res.url}")
except errors.BulkDownloaderException as e:
logger.error(
f"Failed to download resource {res.url} in submission {submission.id} "
@@ -125,6 +186,15 @@ class RedditDownloader(RedditConnector):
return
resource_hash = res.hash.hexdigest()
destination.parent.mkdir(parents=True, exist_ok=True)
# Simple-check: URL-based duplicate detection (fast path)
if self.args.simple_check and hasattr(res, 'url') and res.url in self.url_list:
stored_hash = self.url_list[res.url]
if stored_hash in self.master_hash_list:
logger.info(f"URL {res.url} from submission {submission.id} already downloaded (simple-check)")
return
# Full hash-based duplicate detection
if resource_hash in self.master_hash_list:
if self.args.no_dupes:
logger.info(f"Resource hash {resource_hash} from submission {submission.id} downloaded elsewhere")
@@ -138,11 +208,16 @@ class RedditDownloader(RedditConnector):
f"Hard link made linking {destination} to {self.master_hash_list[resource_hash]}"
f" in submission {submission.id}"
)
files_processed += 1
# Save hash list after successful hard link creation if no_dupes is enabled
if self.args.no_dupes:
self._save_hash_list()
return
try:
with destination.open("wb") as file:
file.write(res.content)
logger.debug(f"Written file to {destination}")
files_processed += 1
except OSError as e:
logger.exception(e)
logger.error(f"Failed to write file in submission {submission.id} to {destination}: {e}")
@@ -150,8 +225,23 @@ class RedditDownloader(RedditConnector):
creation_time = time.mktime(datetime.fromtimestamp(submission.created_utc).timetuple())
os.utime(destination, (creation_time, creation_time))
self.master_hash_list[resource_hash] = destination
# Store URL mapping for simple-check functionality
if hasattr(res, 'url') and self.args.simple_check:
self.url_list[res.url] = resource_hash
logger.debug(f"Hash added to master list: {resource_hash}")
logger.debug(f"Master hash list now contains {len(self.master_hash_list)} entries")
# Save hash list after successful download if no_dupes is enabled
if self.args.no_dupes:
self._save_hash_list()
# Only log "Downloaded submission" if files were actually processed
if files_processed > 0:
logger.info(f"Downloaded submission {submission.id} from {submission.subreddit.display_name}")
else:
logger.info(f"Skipped submission {submission.id} from {submission.subreddit.display_name} (no new files)")
@staticmethod
def scan_existing_files(directory: Path) -> dict[str, Path]:
@@ -166,3 +256,152 @@ class RedditDownloader(RedditConnector):
hash_list = {res[1]: res[0] for res in results}
return hash_list
def get_master_hash_list(self) -> dict[str, Path]:
"""Get the current master hash list for testing purposes."""
return self.master_hash_list
def _load_hash_list(self) -> dict[str, Path]:
"""Load existing hash list from .bdfr_hashes.json in download directory."""
logger.debug(f"Loading hash list from directory: {self.download_directory}")
hash_file_path = self.download_directory / '.bdfr_hashes.json'
if not hash_file_path.exists():
logger.debug(f"No existing hash file found at {hash_file_path}")
return {}
try:
with open(hash_file_path, 'r', encoding='utf-8') as f:
hash_data = json.load(f)
if not isinstance(hash_data, dict):
logger.warning(f"Hash file {hash_file_path} contains invalid data format")
return {}
# Handle new enhanced format
if 'files' in hash_data and isinstance(hash_data['files'], dict):
# New format with enhanced structure
files_data = hash_data['files']
loaded_hashes = {}
urls_data = hash_data.get('urls', {})
for hash_value, file_info in files_data.items():
if isinstance(file_info, dict) and 'path' in file_info:
# New format: {"hash": {"path": "relative/path", "url": "http://..."}}
relative_path = file_info['path']
absolute_path = self.download_directory / relative_path
if absolute_path.exists():
loaded_hashes[hash_value] = absolute_path
else:
logger.debug(f"File {absolute_path} from hash file no longer exists")
# Load URL mapping for simple-check
if 'url' in file_info and file_info['url']:
self.url_list[file_info['url']] = hash_value
elif isinstance(file_info, str):
# Legacy format within new structure: {"hash": "relative/path"}
absolute_path = self.download_directory / file_info
if absolute_path.exists():
loaded_hashes[hash_value] = absolute_path
else:
logger.debug(f"File {absolute_path} from hash file no longer exists")
logger.info(f"Loaded {len(loaded_hashes)} hashes and {len(urls_data)} URLs from enhanced hash file")
return loaded_hashes
else:
# Legacy format: {"hash": "relative/path"}
loaded_hashes = {}
for hash_value, relative_path in hash_data.items():
absolute_path = self.download_directory / relative_path
if absolute_path.exists():
loaded_hashes[hash_value] = absolute_path
else:
logger.debug(f"File {absolute_path} from hash file no longer exists")
logger.info(f"Loaded {len(loaded_hashes)} hashes from legacy hash file")
return loaded_hashes
except json.JSONDecodeError as e:
logger.warning(f"Failed to parse hash file {hash_file_path}: {e}")
return {}
except (OSError, IOError) as e:
logger.warning(f"Failed to read hash file {hash_file_path}: {e}")
return {}
def _save_hash_list(self) -> None:
"""Save current hash list to .bdfr_hashes.json in download directory using atomic write."""
hash_file_path = self.download_directory / '.bdfr_hashes.json'
# Build enhanced data structure for new format
if self.args.simple_check:
# New enhanced format with URLs and metadata
hash_data = {
'files': {},
'urls': self.url_list.copy(),
'metadata': {
'version': '2.0',
'created_with': 'simple_check' if self.args.simple_check else 'standard',
'url_count': len(self.url_list),
'hash_count': len(self.master_hash_list)
}
}
# Convert absolute paths to relative paths for portability
for hash_value, absolute_path in self.master_hash_list.items():
try:
relative_path = absolute_path.relative_to(self.download_directory)
hash_data['files'][hash_value] = {
'path': str(relative_path),
'url': next((url for url, h in self.url_list.items() if h == hash_value), None),
'check_method': 'hash'
}
except ValueError:
# File is not relative to download directory, skip it
logger.debug(f"Skipping file {absolute_path} as it's not relative to download directory")
continue
else:
# Legacy format for backward compatibility
hash_data = {}
for hash_value, absolute_path in self.master_hash_list.items():
try:
relative_path = absolute_path.relative_to(self.download_directory)
hash_data[hash_value] = str(relative_path)
except ValueError:
# File is not relative to download directory, skip it
logger.debug(f"Skipping file {absolute_path} as it's not relative to download directory")
continue
# Atomic write: write to temporary file first, then rename
try:
with tempfile.NamedTemporaryFile(
mode='w',
dir=self.download_directory,
suffix='.tmp',
delete=False,
encoding='utf-8'
) as temp_file:
json.dump(hash_data, temp_file, indent=2)
temp_file_path = temp_file.name
# Atomic rename
if os.name == 'nt': # Windows
# On Windows, we need to remove the target file first if it exists
if hash_file_path.exists():
hash_file_path.unlink()
os.rename(temp_file_path, hash_file_path)
else: # Unix-like systems
os.rename(temp_file_path, hash_file_path)
logger.debug(f"Saved {len(hash_data)} hashes to {hash_file_path}")
except (OSError, IOError) as e:
logger.error(f"Failed to save hash file {hash_file_path}: {e}")
except Exception as e:
logger.error(f"Unexpected error saving hash file {hash_file_path}: {e}")
# Clean up temp file if it still exists
try:
if 'temp_file_path' in locals():
os.unlink(temp_file_path)
except (OSError, IOError):
pass

8
bdfr/oauth2.py
View File

@@ -9,7 +9,9 @@ import socket
from pathlib import Path
import praw
import prawcore
import requests
from praw.util.token_manager import BaseTokenManager
from bdfr.exceptions import BulkDownloaderException, RedditAuthenticationError
@@ -87,13 +89,13 @@ class OAuth2Authenticator:
client.close()
class OAuth2TokenManager(praw.reddit.BaseTokenManager):
class OAuth2TokenManager(BaseTokenManager):
def __init__(self, config: configparser.ConfigParser, config_location: Path):
super(OAuth2TokenManager, self).__init__()
self.config = config
self.config_location = config_location
def pre_refresh_callback(self, authorizer: praw.reddit.Authorizer):
def pre_refresh_callback(self, authorizer: prawcore.auth.BaseAuthorizer):
if authorizer.refresh_token is None:
if self.config.has_option("DEFAULT", "user_token"):
authorizer.refresh_token = self.config.get("DEFAULT", "user_token")
@@ -101,7 +103,7 @@ class OAuth2TokenManager(praw.reddit.BaseTokenManager):
else:
raise RedditAuthenticationError("No auth token loaded in configuration")
def post_refresh_callback(self, authorizer: praw.reddit.Authorizer):
def post_refresh_callback(self, authorizer: prawcore.auth.BaseAuthorizer):
self.config.set("DEFAULT", "user_token", authorizer.refresh_token)
with Path(self.config_location).open(mode="w") as file:
self.config.write(file, True)

2
pyproject.toml
View File

@@ -27,7 +27,7 @@ dependencies = [
"beautifulsoup4>=4.10.0",
"click>=8.0.0",
"dict2xml>=1.7.0",
"praw>=7.2.0",
"praw>=7.8.1",
"pyyaml>=5.4.1",
"requests>=2.25.1",
"yt-dlp>=2022.11.11",

321
tests/test_hash_persistence.py Normal file
View File

@@ -0,0 +1,321 @@
#!/usr/bin/env python3
"""
Test script to verify hash persistence functionality.
"""
import json
import tempfile
import shutil
from pathlib import Path
from unittest.mock import Mock
# Import the necessary modules
import sys
sys.path.insert(0, '/Users/Daniel/Documents/GitHub/bulk-downloader-for-reddit')
from bdfr.configuration import Configuration
from bdfr.downloader import RedditDownloader
def test_hash_persistence():
"""Test the hash persistence functionality."""
# Create a temporary directory for testing
with tempfile.TemporaryDirectory() as temp_dir:
temp_path = Path(temp_dir)
# Create a mock args object
mock_args = Mock()
mock_args.no_dupes = True
mock_args.simple_check = False
mock_args.search_existing = False
mock_args.skip_subreddit = []
mock_args.ignore_user = []
mock_args.min_score = None
mock_args.max_score = None
mock_args.min_score_ratio = None
mock_args.max_score_ratio = None
mock_args.disable_module = []
mock_args.make_hard_links = False
mock_args.max_wait_time = 30
# Create downloader instance
downloader = RedditDownloader.__new__(RedditDownloader)
downloader.args = mock_args
downloader.download_directory = temp_path
downloader.master_hash_list = {}
downloader.excluded_submission_ids = set()
downloader.file_name_formatter = Mock()
downloader.download_filter = Mock()
downloader.download_filter.check_url.return_value = True
downloader.download_filter.check_resource.return_value = True
downloader.authenticator = Mock()
# Test 1: Initially empty hash list
print("Test 1: Loading from empty directory")
hash_list = downloader._load_hash_list()
assert len(hash_list) == 0, f"Expected empty hash list, got {len(hash_list)}"
print("PASS Passed")
# Test 2: Save empty hash list
print("Test 2: Saving empty hash list")
downloader._save_hash_list()
hash_file = temp_path / '.bdfr_hashes.json'
assert hash_file.exists(), "Hash file should be created even when empty"
print("PASS Passed")
# Test 3: Load hash list after saving empty one
print("Test 3: Loading saved empty hash list")
hash_list = downloader._load_hash_list()
assert len(hash_list) == 0, f"Expected empty hash list, got {len(hash_list)}"
print("PASS")
# Test 4: Add some test data and save
print("Test 4: Adding test data and saving")
test_file = temp_path / 'test.txt'
test_file.write_text("test content")
downloader.master_hash_list['test_hash_123'] = test_file
downloader._save_hash_list()
# Verify the saved JSON structure
with open(hash_file, 'r') as f:
saved_data = json.load(f)
assert 'test_hash_123' in saved_data, "Test hash should be in saved data"
assert saved_data['test_hash_123'] == 'test.txt', f"Expected 'test.txt', got {saved_data['test_hash_123']}"
print("PASS Passed")
# Test 5: Load hash list and verify data is restored
print("Test 5: Loading hash list with saved data")
new_downloader = RedditDownloader.__new__(RedditDownloader)
new_downloader.args = mock_args
new_downloader.download_directory = temp_path
new_downloader.master_hash_list = {}
new_downloader.excluded_submission_ids = set()
new_downloader.file_name_formatter = Mock()
new_downloader.download_filter = Mock()
new_downloader.download_filter.check_url.return_value = True
new_downloader.download_filter.check_resource.return_value = True
new_downloader.authenticator = Mock()
loaded_hash_list = new_downloader._load_hash_list()
assert len(loaded_hash_list) == 1, f"Expected 1 hash, got {len(loaded_hash_list)}"
assert 'test_hash_123' in loaded_hash_list, "Test hash should be loaded"
assert loaded_hash_list['test_hash_123'] == test_file, f"File path should match: {loaded_hash_list['test_hash_123']} != {test_file}"
print("PASS Passed")
# Test 6: Test corrupted hash file handling
print("Test 6: Testing corrupted hash file handling")
with open(hash_file, 'w') as f:
f.write("invalid json content")
corrupted_downloader = RedditDownloader.__new__(RedditDownloader)
corrupted_downloader.args = mock_args
corrupted_downloader.download_directory = temp_path
corrupted_downloader.master_hash_list = {}
corrupted_downloader.excluded_submission_ids = set()
corrupted_downloader.file_name_formatter = Mock()
corrupted_downloader.download_filter = Mock()
corrupted_downloader.download_filter.check_url.return_value = True
corrupted_downloader.download_filter.check_resource.return_value = True
corrupted_downloader.authenticator = Mock()
# Should handle corrupted file gracefully and return empty dict
corrupted_hash_list = corrupted_downloader._load_hash_list()
assert len(corrupted_hash_list) == 0, f"Expected empty hash list for corrupted file, got {len(corrupted_hash_list)}"
print("PASS Passed")
print("\nAll tests passed! Hash persistence functionality is working correctly.")
def test_simple_check_functionality():
"""Test the simple-check functionality with URL-based duplicate detection."""
print("\n=== Testing Simple-Check Functionality ===")
# Create a temporary directory for testing
with tempfile.TemporaryDirectory() as temp_dir:
temp_path = Path(temp_dir)
# Create a mock args object with simple_check enabled
mock_args = Mock()
mock_args.no_dupes = True
mock_args.simple_check = True
mock_args.search_existing = False
mock_args.skip_subreddit = []
mock_args.ignore_user = []
mock_args.min_score = None
mock_args.max_score = None
mock_args.min_score_ratio = None
mock_args.max_score_ratio = None
mock_args.disable_module = []
mock_args.make_hard_links = False
mock_args.max_wait_time = 30
# Create downloader instance
downloader = RedditDownloader.__new__(RedditDownloader)
downloader.args = mock_args
downloader.download_directory = temp_path
downloader.master_hash_list = {}
downloader.url_list = {}
downloader.excluded_submission_ids = set()
downloader.file_name_formatter = Mock()
downloader.download_filter = Mock()
downloader.download_filter.check_url.return_value = True
downloader.download_filter.check_resource.return_value = True
downloader.authenticator = Mock()
# Test 1: Initially empty hash list
print("Test 1: Loading from empty directory with simple_check")
hash_list = downloader._load_hash_list()
assert len(hash_list) == 0, f"Expected empty hash list, got {len(hash_list)}"
assert len(downloader.url_list) == 0, f"Expected empty URL list, got {len(downloader.url_list)}"
print("PASS")
# Test 2: Add test data and save with simple_check format
print("Test 2: Adding test data and saving with simple_check format")
test_file = temp_path / 'test.txt'
test_file.write_text("test content")
test_url = "https://example.com/test.txt"
test_hash = "test_hash_123"
downloader.master_hash_list[test_hash] = test_file
downloader.url_list[test_url] = test_hash
downloader._save_hash_list()
# Verify the saved JSON structure has enhanced format
with open(temp_path / '.bdfr_hashes.json', 'r') as f:
saved_data = json.load(f)
assert 'files' in saved_data, "Enhanced format should have 'files' section"
assert 'urls' in saved_data, "Enhanced format should have 'urls' section"
assert 'metadata' in saved_data, "Enhanced format should have 'metadata' section"
assert test_hash in saved_data['files'], "Test hash should be in files section"
assert test_url in saved_data['urls'], "Test URL should be in urls section"
assert saved_data['metadata']['version'] == '2.0', "Version should be 2.0"
assert saved_data['metadata']['created_with'] == 'simple_check', "Should be created with simple_check"
print("PASS")
# Test 3: Load hash list and verify URL mapping is restored
print("Test 3: Loading hash list with URL mappings")
new_downloader = RedditDownloader.__new__(RedditDownloader)
new_downloader.args = mock_args
new_downloader.download_directory = temp_path
new_downloader.master_hash_list = {}
new_downloader.url_list = {}
new_downloader.excluded_submission_ids = set()
new_downloader.file_name_formatter = Mock()
new_downloader.download_filter = Mock()
new_downloader.download_filter.check_url.return_value = True
new_downloader.download_filter.check_resource.return_value = True
new_downloader.authenticator = Mock()
loaded_hash_list = new_downloader._load_hash_list()
assert len(loaded_hash_list) == 1, f"Expected 1 hash, got {len(loaded_hash_list)}"
assert len(new_downloader.url_list) == 1, f"Expected 1 URL, got {len(new_downloader.url_list)}"
assert test_hash in loaded_hash_list, "Test hash should be loaded"
assert test_url in new_downloader.url_list, "Test URL should be loaded"
assert new_downloader.url_list[test_url] == test_hash, "URL should map to correct hash"
print("PASS")
# Test 4: Test URL-based duplicate detection
print("Test 4: Testing URL-based duplicate detection")
# Mock resource with URL that matches our stored URL
mock_resource = Mock()
mock_resource.url = test_url
mock_resource.hash.hexdigest.return_value = test_hash
# Create a mock destination that exists
mock_destination = temp_path / 'existing_file.txt'
mock_destination.parent.mkdir(parents=True, exist_ok=True)
mock_destination.write_text("existing content")
# Mock the file name formatter to return our test destination and resource
downloader.file_name_formatter.format_resource_paths.return_value = [(mock_destination, mock_resource)]
# Mock submission
mock_submission = Mock()
mock_submission.id = "test123"
# This should detect the URL match and skip processing
# We can't easily test the full _download_submission without more mocking,
# but we can verify the URL list is working
assert test_url in downloader.url_list, "URL should be in url_list for duplicate detection"
assert downloader.url_list[test_url] == test_hash, "URL should map to correct hash"
print("PASS")
print("\nAll simple-check tests passed! URL-based duplicate detection is working correctly.")
def test_backward_compatibility():
"""Test that old hash files still work with new implementation."""
print("\n=== Testing Backward Compatibility ===")
# Create a temporary directory for testing
with tempfile.TemporaryDirectory() as temp_dir:
temp_path = Path(temp_dir)
# Create old-format hash file manually
(temp_path / 'relative' / 'path').mkdir(parents=True, exist_ok=True)
(temp_path / 'relative' / 'path' / 'file1.txt').write_text("content1")
(temp_path / 'relative' / 'path' / 'file2.txt').write_text("content2")
old_hash_data = {
"hash1": "relative/path/file1.txt",
"hash2": "relative/path/file2.txt"
}
hash_file = temp_path / '.bdfr_hashes.json'
with open(hash_file, 'w') as f:
json.dump(old_hash_data, f)
# Create downloader and load old format
mock_args = Mock()
mock_args.no_dupes = True
mock_args.simple_check = True # Enable simple_check to test format upgrade
downloader = RedditDownloader.__new__(RedditDownloader)
downloader.args = mock_args
downloader.download_directory = temp_path
downloader.master_hash_list = {}
downloader.url_list = {}
loaded_hashes = downloader._load_hash_list()
assert len(loaded_hashes) == 2, f"Expected 2 hashes from old format, got {len(loaded_hashes)}"
assert "hash1" in loaded_hashes, "hash1 should be loaded from old format"
assert "hash2" in loaded_hashes, "hash2 should be loaded from old format"
assert len(downloader.url_list) == 0, "URL list should be empty for old format"
print("PASS - Old format loaded correctly")
# Test saving in new format
(temp_path / 'another').mkdir(parents=True, exist_ok=True)
test_file = temp_path / 'another' / 'new_file.txt'
test_file.write_text("new content")
downloader.master_hash_list["new_hash"] = test_file
downloader._save_hash_list()
# Verify new format was created
with open(hash_file, 'r') as f:
new_data = json.load(f)
assert 'files' in new_data, "New format should have 'files' section"
assert 'urls' in new_data, "New format should have 'urls' section"
assert 'metadata' in new_data, "New format should have 'metadata' section"
assert new_data['metadata']['version'] == '2.0', "Should be version 2.0"
print("PASS - Old format upgraded to new format correctly")
print("\nBackward compatibility tests passed!")
if __name__ == "__main__":
test_hash_persistence()
test_simple_check_functionality()
test_backward_compatibility()