50 lines
2.1 KiB
Python
50 lines
2.1 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import logging
|
|
from abc import ABC, abstractmethod
|
|
from typing import Optional
|
|
|
|
import requests
|
|
from praw.models import Submission
|
|
|
|
from bdfr.exceptions import ResourceNotFound, SiteDownloaderError
|
|
from bdfr.resource import Resource
|
|
from bdfr.site_authenticator import SiteAuthenticator
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class BaseDownloader(ABC):
|
|
def __init__(self, post: Submission, typical_extension: Optional[str] = None):
|
|
self.post = post
|
|
self.typical_extension = typical_extension
|
|
|
|
@abstractmethod
|
|
def find_resources(self, authenticator: Optional[SiteAuthenticator] = None) -> list[Resource]:
|
|
"""Return list of all un-downloaded Resources from submission"""
|
|
raise NotImplementedError
|
|
|
|
@staticmethod
|
|
def retrieve_url(url: str, cookies: dict = None, headers: dict = None) -> requests.Response:
|
|
max_retries = 3
|
|
for attempt in range(1, max_retries + 1):
|
|
try:
|
|
res = requests.get(url, cookies=cookies, headers=headers, timeout=10)
|
|
if res.status_code != 200:
|
|
logger.error(f"Attempt {attempt}: Server responded with {res.status_code} to {url}")
|
|
if attempt == max_retries:
|
|
raise ResourceNotFound(f"Server responded with {res.status_code} to {url}")
|
|
else:
|
|
return res
|
|
except requests.exceptions.SSLError as ssl_err:
|
|
logger.error(f"Attempt {attempt}: SSL error for {url}: {ssl_err}")
|
|
if attempt == max_retries:
|
|
raise SiteDownloaderError(f"SSL error after {max_retries} attempts for {url}: {ssl_err}")
|
|
except requests.exceptions.RequestException as e:
|
|
logger.error(f"Attempt {attempt}: Request error for {url}: {e}")
|
|
if attempt == max_retries:
|
|
raise SiteDownloaderError(f"Failed to get page {url} after {max_retries} attempts: {e}")
|
|
# Should not reach here
|
|
raise SiteDownloaderError(f"Failed to get page {url} after {max_retries} attempts")
|