Fixed issue where file extenions not found and auth timeout
This commit is contained in:
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test script to verify that the duplicate folder creation fix works correctly.
|
||||
This script simulates the scenario where duplicate posts would previously create empty folders.
|
||||
"""
|
||||
|
||||
import tempfile
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
# Add the bdfr module to the path
|
||||
import sys
|
||||
sys.path.insert(0, '.')
|
||||
|
||||
from bdfr.configuration import Configuration
|
||||
from bdfr.connector import RedditConnector
|
||||
from bdfr.downloader import RedditDownloader
|
||||
|
||||
|
||||
def test_duplicate_folder_creation_fix():
|
||||
"""Test that folders are not created for duplicate posts when no_dupes is enabled."""
|
||||
|
||||
# Create a temporary directory for testing
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
temp_path = Path(temp_dir)
|
||||
|
||||
# Create test configuration
|
||||
args = Configuration()
|
||||
args.no_dupes = True
|
||||
args.folder_scheme = ""
|
||||
args.file_scheme = "{POSTID}"
|
||||
|
||||
# Create downloader instance
|
||||
downloader = RedditDownloader(args)
|
||||
downloader.download_directory = temp_path
|
||||
downloader.file_name_formatter = RedditConnector.create_file_name_formatter(downloader)
|
||||
|
||||
# Mock a submission
|
||||
submission = MagicMock()
|
||||
submission.id = "test123"
|
||||
submission.subreddit.display_name = "testsubreddit"
|
||||
submission.author.name = "testuser"
|
||||
submission.score = 100
|
||||
submission.upvote_ratio = 0.8
|
||||
submission.created_utc = 1640995200 # Jan 1, 2022
|
||||
submission.url = "https://example.com/image.jpg"
|
||||
submission.title = "Test Post"
|
||||
|
||||
# Mock the downloader chain
|
||||
mock_downloader_class = MagicMock()
|
||||
mock_downloader_class.__name__ = "MockDownloader"
|
||||
|
||||
mock_downloader = MagicMock()
|
||||
mock_resource = MagicMock()
|
||||
mock_resource.url = "https://example.com/image.jpg"
|
||||
mock_resource.extension = "jpg"
|
||||
mock_resource.hash.hexdigest.return_value = "duplicate_hash_12345"
|
||||
mock_resource.content = b"fake image content"
|
||||
|
||||
mock_downloader.find_resources.return_value = [mock_resource]
|
||||
|
||||
# Set up the master hash list to contain our "duplicate" hash
|
||||
test_hash = "duplicate_hash_12345"
|
||||
existing_file = temp_path / "existing_file.jpg"
|
||||
existing_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
existing_file.touch()
|
||||
downloader.master_hash_list = {test_hash: existing_file}
|
||||
|
||||
# Mock the download factory
|
||||
import bdfr.site_downloaders.download_factory as df
|
||||
original_pull_lever = df.DownloadFactory.pull_lever
|
||||
df.DownloadFactory.pull_lever = MagicMock(return_value=mock_downloader_class)
|
||||
|
||||
try:
|
||||
# Call the download submission method
|
||||
downloader._download_submission(submission)
|
||||
|
||||
# Check that no new directories were created (the fix)
|
||||
subdirs = [d for d in temp_path.rglob("*") if d.is_dir() and d != temp_path]
|
||||
print(f"Number of subdirectories created: {len(subdirs)}")
|
||||
|
||||
# With the fix, no new directories should be created for duplicates
|
||||
# The only directory that might exist is the one we created for the existing file
|
||||
assert len(subdirs) <= 1, f"Expected 0 or 1 subdirectories, but found {len(subdirs)}"
|
||||
|
||||
print("Test passed: No empty folders created for duplicate posts!")
|
||||
|
||||
finally:
|
||||
# Restore original function
|
||||
df.DownloadFactory.pull_lever = original_pull_lever
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_duplicate_folder_creation_fix()
|
||||
print("All tests passed! The duplicate folder creation fix is working correctly.")
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test extension case normalization functionality
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock
|
||||
from bdfr.resource import Resource
|
||||
|
||||
|
||||
class TestExtensionNormalization:
|
||||
"""Test that extensions are properly normalized to lowercase"""
|
||||
|
||||
def test_url_extensions_normalized(self):
|
||||
"""Test that extensions from URLs are normalized to lowercase"""
|
||||
test_cases = [
|
||||
("https://example.com/image.JPG", ".jpg"),
|
||||
("https://example.com/image.jpeg", ".jpeg"),
|
||||
("https://example.com/image.JPEG", ".jpeg"),
|
||||
("https://example.com/image.jpg", ".jpg"),
|
||||
("https://example.com/image.PNG", ".png"),
|
||||
("https://example.com/image.GIF", ".gif"),
|
||||
]
|
||||
|
||||
for url, expected in test_cases:
|
||||
mock_submission = MagicMock()
|
||||
mock_submission.id = "test123"
|
||||
|
||||
resource = Resource(mock_submission, url, lambda: None)
|
||||
assert resource.extension == expected, f"URL {url} should normalize to {expected}, got {resource.extension}"
|
||||
|
||||
def test_reddit_media_urls_normalized(self):
|
||||
"""Test that Reddit media URLs are properly normalized"""
|
||||
test_cases = [
|
||||
("https://www.reddit.com/media?url=https%3A%2F%2Fi.redd.it%2Fr2mv10i4vkfd1.JPG", ".jpg"),
|
||||
("https://www.reddit.com/media?url=https%3A%2F%2Fi.redd.it%2Fr2mv10i4vkfd1.jpeg", ".jpeg"),
|
||||
("https://www.reddit.com/media?url=https%3A%2F%2Fi.redd.it%2Fr2mv10i4vkfd1.JPEG", ".jpeg"),
|
||||
("https://www.reddit.com/media?url=https%3A%2F%2Fi.redd.it%2Fr2mv10i4vkfd1.PNG", ".png"),
|
||||
]
|
||||
|
||||
for url, expected in test_cases:
|
||||
mock_submission = MagicMock()
|
||||
mock_submission.id = "test123"
|
||||
|
||||
resource = Resource(mock_submission, url, lambda: None)
|
||||
assert resource.extension == expected, f"Reddit media URL {url} should normalize to {expected}, got {resource.extension}"
|
||||
|
||||
def test_constructor_extensions_normalized(self):
|
||||
"""Test that extensions passed to constructor are normalized"""
|
||||
test_cases = [
|
||||
(".JPG", ".jpg"),
|
||||
(".JPEG", ".jpeg"),
|
||||
(".PNG", ".png"),
|
||||
(".GIF", ".gif"),
|
||||
]
|
||||
|
||||
for input_ext, expected in test_cases:
|
||||
mock_submission = MagicMock()
|
||||
mock_submission.id = "test123"
|
||||
|
||||
resource = Resource(mock_submission, "https://example.com/test", lambda: None, input_ext)
|
||||
assert resource.extension == expected, f"Constructor extension {input_ext} should normalize to {expected}, got {resource.extension}"
|
||||
|
||||
def test_magic_number_detection_normalized(self):
|
||||
"""Test that magic number detection returns normalized extensions"""
|
||||
mock_submission = MagicMock()
|
||||
mock_submission.id = "test123"
|
||||
|
||||
# Test JPEG magic number detection
|
||||
jpeg_content = b'\xFF\xD8\xFF' + b'0' * 100 # JPEG magic number
|
||||
resource = Resource(mock_submission, "https://example.com/no-extension", lambda params: jpeg_content)
|
||||
resource.download() # Trigger content-based detection
|
||||
assert resource.extension == ".jpg", f"Magic number detection should return .jpg, got {resource.extension}"
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test script to debug file extension detection issues
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
from bdfr.resource import Resource
|
||||
|
||||
def test_extension_detection():
|
||||
"""Test extension detection with various URL patterns"""
|
||||
|
||||
test_cases = [
|
||||
# Standard URLs with extensions
|
||||
("https://example.com/image.jpg", ".jpg"),
|
||||
("https://example.com/video.mp4", ".mp4"),
|
||||
("https://files.example.com/document.pdf", ".pdf"),
|
||||
|
||||
# URLs without extensions
|
||||
("https://example.com/api/data", None),
|
||||
("https://example.com/path/without/extension", None),
|
||||
|
||||
# URLs with query parameters
|
||||
("https://example.com/image.jpg?size=large", ".jpg"),
|
||||
("https://example.com/video.mp4?utm_source=test", ".mp4"),
|
||||
|
||||
# URLs with fragments
|
||||
("https://example.com/image.png#section", ".png"),
|
||||
|
||||
# Complex paths
|
||||
("https://imgur.com/a/gallery123", None),
|
||||
("https://reddit.com/r/test/abc123_def456_789", None),
|
||||
|
||||
# Edge cases that might cause weird names
|
||||
("https://example.com/L7SW9E~G", None),
|
||||
("https://example.com/temp/file", None),
|
||||
|
||||
# Reddit media URLs (the actual issue)
|
||||
("https://www.reddit.com/media?url=https%3A%2F%2Fi.redd.it%2Fr2mv10i4vkfd1.jpeg", ".jpeg"),
|
||||
("https://i.redd.it/r2mv10i4vkfd1.jpeg", ".jpeg"),
|
||||
]
|
||||
|
||||
print("Testing extension detection with various URLs:")
|
||||
print("=" * 60)
|
||||
|
||||
for url, expected in test_cases:
|
||||
# Create a mock submission
|
||||
mock_submission = MagicMock()
|
||||
mock_submission.id = "test123"
|
||||
|
||||
# Create resource and test extension detection
|
||||
resource = Resource(mock_submission, url, lambda: None)
|
||||
|
||||
print(f"URL: {url}")
|
||||
print(f"Expected: {expected}")
|
||||
print(f"Detected: {resource.extension}")
|
||||
print(f"Match: {'YES' if resource.extension == expected else 'NO'}")
|
||||
print("-" * 40)
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_extension_detection()
|
||||
Reference in New Issue
Block a user