ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #419 from Serene-Arc/bug_fix_407

Browse Source
This commit is contained in:
Serene
2021-05-23 12:30:20 +10:00
committed by GitHub
parent 5aae6b3df8 1b23e38ce4
commit a511c706c7
2 changed files with 11 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
bdfr/downloader.py
View File

@@ -22,13 +22,13 @@ logger = logging.getLogger(__name__)
def _calc_hash(existing_file: Path): def _calc_hash(existing_file: Path):
CHUNK_SIZE = 1024 * 1024 chunk_size = 1024 * 1024
md5_hash = hashlib.md5() md5_hash = hashlib.md5()
with open(existing_file, 'rb') as file: with open(existing_file, 'rb') as file:
chunk = file.read(CHUNK_SIZE) chunk = file.read(chunk_size)
while chunk: while chunk:
md5_hash.update(chunk) md5_hash.update(chunk)
chunk = file.read(CHUNK_SIZE) chunk = file.read(chunk_size)
file_hash = md5_hash.hexdigest() file_hash = md5_hash.hexdigest()
return existing_file, file_hash return existing_file, file_hash
@@ -94,9 +94,13 @@ class RedditDownloader(RedditConnector):
f'Hard link made linking {destination} to {self.master_hash_list[resource_hash]}' f'Hard link made linking {destination} to {self.master_hash_list[resource_hash]}'
f' in submission {submission.id}') f' in submission {submission.id}')
return return
with open(destination, 'wb') as file: try:
file.write(res.content) with open(destination, 'wb') as file:
logger.debug(f'Written file to {destination}') file.write(res.content)
logger.debug(f'Written file to {destination}')
except OSError as e:
logger.exception(e)
logger.error(f'Failed to write file to {destination} in submission {submission.id}: {e}')
creation_time = time.mktime(datetime.fromtimestamp(submission.created_utc).timetuple()) creation_time = time.mktime(datetime.fromtimestamp(submission.created_utc).timetuple())
os.utime(destination, (creation_time, creation_time)) os.utime(destination, (creation_time, creation_time))
self.master_hash_list[resource_hash] = destination self.master_hash_list[resource_hash] = destination

1
scripts/extract_failed_ids.sh
View File

@@ -18,4 +18,5 @@ fi
grep 'Could not download submission' "$file" | awk '{ print $12 }' | rev | cut -c 2- | rev ; grep 'Could not download submission' "$file" | awk '{ print $12 }' | rev | cut -c 2- | rev ;
grep 'Failed to download resource' "$file" | awk '{ print $15 }' ; grep 'Failed to download resource' "$file" | awk '{ print $15 }' ;
grep 'failed to download submission' "$file" | awk '{ print $14 }' | rev | cut -c 2- | rev ; grep 'failed to download submission' "$file" | awk '{ print $14 }' | rev | cut -c 2- | rev ;
grep 'Failed to write file' "$file" | awk '{ print $16 }' | rev | cut -c 2- | rev ;
} >>"$output" } >>"$output"