ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify method structure

Browse Source
This commit is contained in:
Serene-Arc
2021-05-21 16:50:05 +10:00
committed by Serene
parent da8c64ec51
commit a104a154fc
1 changed files with 27 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
bdfr/downloader.py
View File

@@ -71,35 +71,36 @@ class RedditDownloader(RedditConnector):
for destination, res in self.file_name_formatter.format_resource_paths(content, self.download_directory): for destination, res in self.file_name_formatter.format_resource_paths(content, self.download_directory):
if destination.exists(): if destination.exists():
logger.debug(f'File {destination} already exists, continuing') logger.debug(f'File {destination} already exists, continuing')
continue
elif not self.download_filter.check_resource(res): elif not self.download_filter.check_resource(res):
logger.debug(f'Download filter removed {submission.id} with URL {submission.url}') logger.debug(f'Download filter removed {submission.id} with URL {submission.url}')
else: continue
try: try:
res.download(self.args.max_wait_time) res.download(self.args.max_wait_time)
except errors.BulkDownloaderException as e: except errors.BulkDownloaderException as e:
logger.error(f'Failed to download resource {res.url} in submission {submission.id} ' logger.error(f'Failed to download resource {res.url} in submission {submission.id} '
f'with downloader {downloader_class.__name__}: {e}') f'with downloader {downloader_class.__name__}: {e}')
return
resource_hash = res.hash.hexdigest()
destination.parent.mkdir(parents=True, exist_ok=True)
if resource_hash in self.master_hash_list:
if self.args.no_dupes:
logger.info(
f'Resource hash {resource_hash} from submission {submission.id} downloaded elsewhere')
return return
resource_hash = res.hash.hexdigest() elif self.args.make_hard_links:
destination.parent.mkdir(parents=True, exist_ok=True) self.master_hash_list[resource_hash].link_to(destination)
if resource_hash in self.master_hash_list: logger.info(
if self.args.no_dupes: f'Hard link made linking {destination} to {self.master_hash_list[resource_hash]}')
logger.info( return
f'Resource hash {resource_hash} from submission {submission.id} downloaded elsewhere') with open(destination, 'wb') as file:
return file.write(res.content)
elif self.args.make_hard_links: logger.debug(f'Written file to {destination}')
self.master_hash_list[resource_hash].link_to(destination) creation_time = time.mktime(datetime.fromtimestamp(submission.created_utc).timetuple())
logger.info( os.utime(destination, (creation_time, creation_time))
f'Hard link made linking {destination} to {self.master_hash_list[resource_hash]}') self.master_hash_list[resource_hash] = destination
return logger.debug(f'Hash added to master list: {resource_hash}')
with open(destination, 'wb') as file: logger.info(f'Downloaded submission {submission.id} from {submission.subreddit.display_name}')
file.write(res.content)
logger.debug(f'Written file to {destination}')
creation_time = time.mktime(datetime.fromtimestamp(submission.created_utc).timetuple())
os.utime(destination, (creation_time, creation_time))
self.master_hash_list[resource_hash] = destination
logger.debug(f'Hash added to master list: {resource_hash}')
logger.info(f'Downloaded submission {submission.id} from {submission.subreddit.display_name}')
@staticmethod @staticmethod
def scan_existing_files(directory: Path) -> dict[str, Path]: def scan_existing_files(directory: Path) -> dict[str, Path]: