ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add existence checking

Browse Source
This commit is contained in:
Serene-Arc
2021-02-14 18:52:04 +10:00
committed by Ali Parlakci
parent 722e6cb73a
commit e646ae4a84
1 changed files with 11 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
bulkredditdownloader/downloader.py
View File

@@ -176,7 +176,6 @@ class RedditDownloader:
self._download_submission(submission) self._download_submission(submission)
def _download_submission(self, submission: praw.models.Submission): def _download_submission(self, submission: praw.models.Submission):
# TODO: check existence here
if self.download_filter.check_url(submission.url): if self.download_filter.check_url(submission.url):
try: try:
downloader_class = DownloadFactory.pull_lever(submission.url) downloader_class = DownloadFactory.pull_lever(submission.url)
@@ -184,13 +183,17 @@ class RedditDownloader:
content = downloader.download() content = downloader.download()
for res in content: for res in content:
destination = self.file_name_formatter.format_path(res, self.download_directory) destination = self.file_name_formatter.format_path(res, self.download_directory)
if res.hash.hexdigest() not in self.master_hash_list: if destination.exists():
destination.parent.mkdir(parents=True, exist_ok=True) logger.debug('File already exists: {}'.format(destination))
with open(destination, 'wb') as file: else:
file.write(res.content) if res.hash.hexdigest() not in self.master_hash_list:
logger.debug('Written file to {}'.format(destination)) # TODO: consider making a hard link/symlink here
self.master_hash_list.append(res.hash.hexdigest()) destination.parent.mkdir(parents=True, exist_ok=True)
logger.debug('Hash added to master list: {}'.format(res.hash.hexdigest())) with open(destination, 'wb') as file:
file.write(res.content)
logger.debug('Written file to {}'.format(destination))
self.master_hash_list.append(res.hash.hexdigest())
logger.debug('Hash added to master list: {}'.format(res.hash.hexdigest()))
logger.info('Downloaded submission {}'.format(submission.name)) logger.info('Downloaded submission {}'.format(submission.name))
except NotADownloadableLinkError as e: except NotADownloadableLinkError as e: