ModerateWinGuy/BDFR_Web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #626 from chapmanjacobd/patch-1

Browse Source
This commit is contained in:
Serene
2022-05-08 12:09:54 +10:00
committed by GitHub
parent 90b680935e 5775c0ab9f
commit c410682cc8
4 changed files with 12 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
scripts/extract_failed_ids.sh
View File

@@ -7,17 +7,10 @@ else
exit 1 exit 1
fi fi
if [ -n "$2" ]; then
output="$2"
echo "Outputting IDs to $output"
else
output="./failed.txt"
fi
{ {
grep 'Could not download submission' "$file" | awk '{ print $12 }' | rev | cut -c 2- | rev ; grep 'Could not download submission' "$file" | awk '{ print $12 }' | rev | cut -c 2- | rev ;
grep 'Failed to download resource' "$file" | awk '{ print $15 }' ; grep 'Failed to download resource' "$file" | awk '{ print $15 }' ;
grep 'failed to download submission' "$file" | awk '{ print $14 }' | rev | cut -c 2- | rev ; grep 'failed to download submission' "$file" | awk '{ print $14 }' | rev | cut -c 2- | rev ;
grep 'Failed to write file' "$file" | awk '{ print $14 }' ; grep 'Failed to write file' "$file" | awk '{ print $14 }' ;
grep 'skipped due to disabled module' "$file" | awk '{ print $9 }' ; grep 'skipped due to disabled module' "$file" | awk '{ print $9 }' ;
} >>"$output" }

9
scripts/extract_successful_ids.sh
View File

@@ -7,17 +7,10 @@ else
exit 1 exit 1
fi fi
if [ -n "$2" ]; then
output="$2"
echo "Outputting IDs to $output"
else
output="./successful.txt"
fi
{ {
grep 'Downloaded submission' "$file" | awk '{ print $(NF-2) }' ; grep 'Downloaded submission' "$file" | awk '{ print $(NF-2) }' ;
grep 'Resource hash' "$file" | awk '{ print $(NF-2) }' ; grep 'Resource hash' "$file" | awk '{ print $(NF-2) }' ;
grep 'Download filter' "$file" | awk '{ print $(NF-3) }' ; grep 'Download filter' "$file" | awk '{ print $(NF-3) }' ;
grep 'already exists, continuing' "$file" | awk '{ print $(NF-3) }' ; grep 'already exists, continuing' "$file" | awk '{ print $(NF-3) }' ;
grep 'Hard link made' "$file" | awk '{ print $(NF) }' ; grep 'Hard link made' "$file" | awk '{ print $(NF) }' ;
} >> "$output" }

10
scripts/tests/test_extract_failed_ids.bats
View File

@@ -13,31 +13,31 @@ teardown() {
} }
@test "fail no downloader module" { @test "fail no downloader module" {
run ../extract_failed_ids.sh ./example_logfiles/failed_no_downloader.txt run ../extract_failed_ids.sh ./example_logfiles/failed_no_downloader.txt >> failed.txt
assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "3" ]; assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "3" ];
assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ]; assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ];
} }
@test "fail resource error" { @test "fail resource error" {
run ../extract_failed_ids.sh ./example_logfiles/failed_resource_error.txt run ../extract_failed_ids.sh ./example_logfiles/failed_resource_error.txt >> failed.txt
assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "1" ]; assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "1" ];
assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ]; assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ];
} }
@test "fail site downloader error" { @test "fail site downloader error" {
run ../extract_failed_ids.sh ./example_logfiles/failed_sitedownloader_error.txt run ../extract_failed_ids.sh ./example_logfiles/failed_sitedownloader_error.txt >> failed.txt
assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "2" ]; assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "2" ];
assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ]; assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ];
} }
@test "fail failed file write" { @test "fail failed file write" {
run ../extract_failed_ids.sh ./example_logfiles/failed_write_error.txt run ../extract_failed_ids.sh ./example_logfiles/failed_write_error.txt >> failed.txt
assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "1" ]; assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "1" ];
assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ]; assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ];
} }
@test "fail disabled module" { @test "fail disabled module" {
run ../extract_failed_ids.sh ./example_logfiles/failed_disabled_module.txt run ../extract_failed_ids.sh ./example_logfiles/failed_disabled_module.txt >> failed.txt
assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "1" ]; assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "1" ];
assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ]; assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ];
} }

10
scripts/tests/test_extract_successful_ids.bats
View File

@@ -8,31 +8,31 @@ teardown() {
} }
@test "success downloaded submission" { @test "success downloaded submission" {
run ../extract_successful_ids.sh ./example_logfiles/succeed_downloaded_submission.txt run ../extract_successful_ids.sh ./example_logfiles/succeed_downloaded_submission.txt >> ./successful.txt
assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "7" ]; assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "7" ];
assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ]; assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ];
} }
@test "success resource hash" { @test "success resource hash" {
run ../extract_successful_ids.sh ./example_logfiles/succeed_resource_hash.txt run ../extract_successful_ids.sh ./example_logfiles/succeed_resource_hash.txt >> ./successful.txt
assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "1" ]; assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "1" ];
assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ]; assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ];
} }
@test "success download filter" { @test "success download filter" {
run ../extract_successful_ids.sh ./example_logfiles/succeed_download_filter.txt run ../extract_successful_ids.sh ./example_logfiles/succeed_download_filter.txt >> ./successful.txt
assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "3" ]; assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "3" ];
assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ]; assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ];
} }
@test "success already exists" { @test "success already exists" {
run ../extract_successful_ids.sh ./example_logfiles/succeed_already_exists.txt run ../extract_successful_ids.sh ./example_logfiles/succeed_already_exists.txt >> ./successful.txt
assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "3" ]; assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "3" ];
assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ]; assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ];
} }
@test "success hard link" { @test "success hard link" {
run ../extract_successful_ids.sh ./example_logfiles/succeed_hard_link.txt run ../extract_successful_ids.sh ./example_logfiles/succeed_hard_link.txt >> ./successful.txt
assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "1" ]; assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "1" ];
assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ]; assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ];
} }