ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Re-implement --no-dupes flag

Browse Source
This commit is contained in:
Serene-Arc
2021-02-14 19:22:31 +10:00
committed by Ali Parlakci
parent eac2381a0a
commit 4bbe41a2f8
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bulkredditdownloader/downloader.py
View File

@@ -186,7 +186,7 @@ class RedditDownloader:
if destination.exists(): if destination.exists():
logger.debug('File already exists: {}'.format(destination)) logger.debug('File already exists: {}'.format(destination))
else: else:
if res.hash.hexdigest() not in self.master_hash_list: if res.hash.hexdigest() not in self.master_hash_list and self.args.no_dupes:
# TODO: consider making a hard link/symlink here # TODO: consider making a hard link/symlink here
destination.parent.mkdir(parents=True, exist_ok=True) destination.parent.mkdir(parents=True, exist_ok=True)
with open(destination, 'wb') as file: with open(destination, 'wb') as file:
@@ -194,6 +194,8 @@ class RedditDownloader:
logger.debug('Written file to {}'.format(destination)) logger.debug('Written file to {}'.format(destination))
self.master_hash_list.append(res.hash.hexdigest()) self.master_hash_list.append(res.hash.hexdigest())
logger.debug('Hash added to master list: {}'.format(res.hash.hexdigest())) logger.debug('Hash added to master list: {}'.format(res.hash.hexdigest()))
else:
logger.debug(f'Resource from {res.url} downloaded elsewhere')
logger.info('Downloaded submission {}'.format(submission.name)) logger.info('Downloaded submission {}'.format(submission.name))
except NotADownloadableLinkError as e: except NotADownloadableLinkError as e: