From b921d03705b3f47a995ddbff47f08d60e34de362 Mon Sep 17 00:00:00 2001 From: Jacob Chapman <7908073+chapmanjacobd@users.noreply.github.com> Date: Mon, 18 Apr 2022 00:30:17 -0500 Subject: [PATCH 1/4] Use stdout --- scripts/extract_failed_ids.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/scripts/extract_failed_ids.sh b/scripts/extract_failed_ids.sh index 8addf7e..64d1e72 100755 --- a/scripts/extract_failed_ids.sh +++ b/scripts/extract_failed_ids.sh @@ -7,17 +7,10 @@ else exit 1 fi -if [ -n "$2" ]; then - output="$2" - echo "Outputting IDs to $output" -else - output="./failed.txt" -fi - { grep 'Could not download submission' "$file" | awk '{ print $12 }' | rev | cut -c 2- | rev ; grep 'Failed to download resource' "$file" | awk '{ print $15 }' ; grep 'failed to download submission' "$file" | awk '{ print $14 }' | rev | cut -c 2- | rev ; grep 'Failed to write file' "$file" | awk '{ print $14 }' ; grep 'skipped due to disabled module' "$file" | awk '{ print $9 }' ; -} >>"$output" +} From 4e050c50d6503370af61c1e67886a87f9d3635ce Mon Sep 17 00:00:00 2001 From: Jacob Chapman <7908073+chapmanjacobd@users.noreply.github.com> Date: Mon, 18 Apr 2022 20:42:50 -0500 Subject: [PATCH 2/4] okay --- scripts/tests/test_extract_failed_ids.bats | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/tests/test_extract_failed_ids.bats b/scripts/tests/test_extract_failed_ids.bats index 75b9bff..a716cba 100644 --- a/scripts/tests/test_extract_failed_ids.bats +++ b/scripts/tests/test_extract_failed_ids.bats @@ -13,31 +13,31 @@ teardown() { } @test "fail no downloader module" { - run ../extract_failed_ids.sh ./example_logfiles/failed_no_downloader.txt + run ../extract_failed_ids.sh ./example_logfiles/failed_no_downloader.txt >> failed.txt assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "3" ]; assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ]; } @test "fail resource error" { - run ../extract_failed_ids.sh ./example_logfiles/failed_resource_error.txt + run ../extract_failed_ids.sh ./example_logfiles/failed_resource_error.txt >> failed.txt assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "1" ]; assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ]; } @test "fail site downloader error" { - run ../extract_failed_ids.sh ./example_logfiles/failed_sitedownloader_error.txt + run ../extract_failed_ids.sh ./example_logfiles/failed_sitedownloader_error.txt >> failed.txt assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "2" ]; assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ]; } @test "fail failed file write" { - run ../extract_failed_ids.sh ./example_logfiles/failed_write_error.txt + run ../extract_failed_ids.sh ./example_logfiles/failed_write_error.txt >> failed.txt assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "1" ]; assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ]; } @test "fail disabled module" { - run ../extract_failed_ids.sh ./example_logfiles/failed_disabled_module.txt + run ../extract_failed_ids.sh ./example_logfiles/failed_disabled_module.txt >> failed.txt assert [ "$( wc -l 'failed.txt' | awk '{ print $1 }' )" -eq "1" ]; assert [ "$( grep -Ecv '\w{6,7}' 'failed.txt' )" -eq "0" ]; } From 484bde9b136f18a02d9629b655bb3d9405848719 Mon Sep 17 00:00:00 2001 From: Jacob Chapman <7908073+chapmanjacobd@users.noreply.github.com> Date: Mon, 18 Apr 2022 20:47:35 -0500 Subject: [PATCH 3/4] oh there's another one --- scripts/extract_successful_ids.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/scripts/extract_successful_ids.sh b/scripts/extract_successful_ids.sh index 011ba6c..e8f482e 100755 --- a/scripts/extract_successful_ids.sh +++ b/scripts/extract_successful_ids.sh @@ -7,17 +7,10 @@ else exit 1 fi -if [ -n "$2" ]; then - output="$2" - echo "Outputting IDs to $output" -else - output="./successful.txt" -fi - { grep 'Downloaded submission' "$file" | awk '{ print $(NF-2) }' ; grep 'Resource hash' "$file" | awk '{ print $(NF-2) }' ; grep 'Download filter' "$file" | awk '{ print $(NF-3) }' ; grep 'already exists, continuing' "$file" | awk '{ print $(NF-3) }' ; grep 'Hard link made' "$file" | awk '{ print $(NF) }' ; -} >> "$output" +} From 5775c0ab9f19a05cebaf0cf687a7eca1eeefd87c Mon Sep 17 00:00:00 2001 From: Jacob Chapman <7908073+chapmanjacobd@users.noreply.github.com> Date: Mon, 18 Apr 2022 20:48:02 -0500 Subject: [PATCH 4/4] and this one too --- scripts/tests/test_extract_successful_ids.bats | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/tests/test_extract_successful_ids.bats b/scripts/tests/test_extract_successful_ids.bats index 364bedb..caa8dd1 100644 --- a/scripts/tests/test_extract_successful_ids.bats +++ b/scripts/tests/test_extract_successful_ids.bats @@ -8,31 +8,31 @@ teardown() { } @test "success downloaded submission" { - run ../extract_successful_ids.sh ./example_logfiles/succeed_downloaded_submission.txt + run ../extract_successful_ids.sh ./example_logfiles/succeed_downloaded_submission.txt >> ./successful.txt assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "7" ]; assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ]; } @test "success resource hash" { - run ../extract_successful_ids.sh ./example_logfiles/succeed_resource_hash.txt + run ../extract_successful_ids.sh ./example_logfiles/succeed_resource_hash.txt >> ./successful.txt assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "1" ]; assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ]; } @test "success download filter" { - run ../extract_successful_ids.sh ./example_logfiles/succeed_download_filter.txt + run ../extract_successful_ids.sh ./example_logfiles/succeed_download_filter.txt >> ./successful.txt assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "3" ]; assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ]; } @test "success already exists" { - run ../extract_successful_ids.sh ./example_logfiles/succeed_already_exists.txt + run ../extract_successful_ids.sh ./example_logfiles/succeed_already_exists.txt >> ./successful.txt assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "3" ]; assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ]; } @test "success hard link" { - run ../extract_successful_ids.sh ./example_logfiles/succeed_hard_link.txt + run ../extract_successful_ids.sh ./example_logfiles/succeed_hard_link.txt >> ./successful.txt assert [ "$( wc -l 'successful.txt' | awk '{ print $1 }' )" -eq "1" ]; assert [ "$( grep -Ecv '\w{6,7}' 'successful.txt' )" -eq "0" ]; }