#! /bin/bash
#
# Warning: modifying this script would very likely defeat the purpose of having
# it supplied to you.
#
# This script can be used to test and grade your solution for the MIPS assembler
# assignment. This is the script we will use to do that, so if your tar file
# does not work for you with this script, it won't work for us either.
# That would be unfortunate.
#
# Invoke as: ./gradeC02.sh [ -m1 | -m2 | -final ] <name of submission tar file>
# -m1 selects the tests for milestone 1
# -m2 selects the tests for milestone 2
# -final selects the tests for the final submission
# -ec selects the tests for the extra-credit option (only)
# We suggest you name your tar file PID.tar, where PID is
# your VT email pid (e.g., wmcquain.tar).
#
#
# Last modified: Sept 26, 2021
#
# To use this test script:
# - Untar the distributed testing tar file into a directory; that should
# create the following directory structure:
#
# ./gradeC02.sh - this script file
# ./compare - 64-bit CentOS 8 utility for checking results
# ./milestone1/m1testxx.asm - MIPS files to be assembled
# ./milestone1/m1testxx.o - reference translations
# ./milestone2/m2testxx.asm - MIPS files to be assembled
# ./milestone2/m2testxx.o - reference translations
# ./final/ftestxx.asm - MIPS files to be assembled
# ./final/ftestxx.o - reference translations
#
# We will refer to the extraction directory as the test directory.
# - Set execute permissions for grade.sh.
# - Prepare the tar file you intend to submit and place it into the same
# directory as gradeC02.sh.
# - Execute the grading script command as described above.
#
# gradeC03.sh will create a subdirectory, ./build, unpack your submission tar
# file into it, and use your makefile to build the specified target,
# "assemble".
#
# If there is no makefile, the script will exit with an error message.
#
# The script will then check for the existence of the specified executable,
# "assemble". If there is no such file, the script will exit with an error
# message. Otherwise, the script will move the executable "assemble" into
# the test directory, and execute it on each of the asm files in the
# appropriate directory. For each test case, the script will use the compare
# utility to compare your translation to the reference translation, producing
# a score for each test case.
#
# If -final is selected, there is additional testing:
# - symbol table generation is tested
# - Valgrind analysis is run on a single test case
# - a test is run for the extra-credit option
#
# Exit codes:
# 0 - normal execution; no major errors were detected
# 1 - wrong number of command-line parameters
# 2 - invalid switch used on command-line
# 3 - something is missing from test file directory
# 4 - file named on command line does not exist
# 5 - file named on command line is not a tar file
# 6 - tar file does not contain a makefile
# 7 - build failed
# 8 - build produced nonexecutable file (unlikely)
#
# Configuration variables:
#
# Names for directories
buildDir="build"
# Flag for testing control
runEC="no"
# Names for log files created by this script:
buildLog="buildLog.txt"
testLog="testLog.txt"
# Name for the specified make target and executable
makeTarget="assembler"
exeName="assemble"
# Delimiter to separate sections of report file:
Separator="============================================================"
# Set results file names
testLog=testing_details.txt
summaryLog=testing_summary.txt
##################################### fn to check existence of test data
# param1: -m1 or -m2 or -final or -ec
testsOK() {
if [[ $1 == "-m1" ]]; then
nCases=3
testDataDir="milestone1"
# Set array of names of test input files
asmFileNames=("m1test01.asm" "m1test02.asm" "m1test03.asm")
# Set array of names of reference object files
refFileNames=("m1test01.o" "m1test02.o" "m1test03.o")
# Set array of names of assembler-produced student object files
stuFileNames=("m1stu01.o" "m1stu02.o" "m1stu03.o")
elif [[ $1 == "-m2" ]]; then
nCases=5
testDataDir="milestone2"
# Set array of names of test input files
asmFileNames=("m2test01.asm" "m2test02.asm" "m2test03.asm" "m2test04.asm" "m2test05.asm")
# Set array of names of reference object files
refFileNames=("m2test01.o" "m2test02.o" "m2test03.o" "m2test04.o" "m2test05.o")
# Set array of names of assembler-produced student object files
stuFileNames=("m2stu01.o" "m2stu02.o" "m2stu03.o" "m2stu04.o" "m2stu05.o")
elif [[ $1 == "-final" ]]; then
nCases=10
testDataDir="final"
# Set array of names of test input files
asmFileNames=("ftest01.asm" "ftest02.asm" "ftest03.asm" "ftest04.asm" "ftest05.asm" "ftest06.asm" "ftest07.asm" "ftest08.asm" "ftest09.asm" "ftest10.asm")
# Set array of names of reference object files
refFileNames=("ftest01.o" "ftest02.o" "ftest03.o" "ftest04.o" "ftest05.o" "ftest06.o" "ftest07.o" "ftest08.o" "ftest09.o" "ftest10.o")
# Set array of names of assembler-produced student object files
stuFileNames=("fstu01.o" "fstu02.o" "fstu03.o" "fstu04.o" "fstu05.o" "fstu06.o" "fstu07.o" "fstu08.o" "fstu09.o" "fstu10.o")
else
echo "Unknown option: $1"
return 1
fi
if [[ ! -d ./$testDataDir ]]; then
echo "Test data directory does not exist!"
echo "Download a fresh copy of the tar file and try again."
return 1
fi
for fName in ${asmFileNames[@]}
do
if [[ ! -e "./$testDataDir/$fName" ]]; then
echo "$fName is not in the test data directory!"
echo "Download a fresh copy of the tar file and try again."
return 2
fi
done
for fName in ${refFileNames[@]}
do
if [[ ! -e "./$testDataDir/$fName" ]]; then
echo "$fName is not in the test data directory!"
echo "Download a fresh copy of the tar file and try again."
return 2
fi
done
return 0
}
############################################# fn to check for tar file
# param1: name of file to be checked
isTar() {
mimeType=`file -b --mime-type $1`
if [[ $mimeType == "application/x-tar" ]]; then
return 0
fi
if [[ $mimeType == "application/x-gzip" ]]; then
return 0
fi
return 1
}
##################################### fn to extract token from file name
# param1: (possibly fully-qualified) name of file
# Note: in production use, the first part of the file name will be the
# student's PID
#
getPID() {
fname=$1
# strip off any leading path info
fname=${fname##*/}
# extract first token of file name
spid=${fname%%.*}
}
###################################### clean up old test files
clean() {
rm -Rf build # delete build directory
rm -f *.o # delete old student object files, or stray ones
rm -f *.txt # delete any old report files
rm -f assemble # delete old assembler executable
exit 0
}
############################################################ Choose test data directory
# Check number of command-line parameters
if [[ $# -eq 1 && $1 == "-clean" ]]; then
clean
fi
if [[ $# -ne 2 ]]; then
echo "Invocations: gradeC02.sh [-m1 | -m2 | -final] [name of submitted tar file]"
echo " gradeC02.sh -clean"
exit 1
fi
if [[ $1 == "-m1" ]]; then
echo "Running tests for milestone 1."
testDataDir="./milestone1"
elif [[ $1 == "-m2" ]]; then
echo "Running tests for milestone 2."
testDataDir="./milestone2"
elif [[ $1 == "-final" ]]; then
echo "Running tests for final submission."
testDataDir="./final"
else
echo "Unknown option: $1"
exit 2
fi
############################################################ Validate environment
# Check for valid test data subdirectory:
testsOK $1
if [[ ! $? -eq 0 ]]; then
echo "The test data directory does not exist, or its contents are not valid."
echo "Fix this error."
exit 3
fi
############################################################ Validate command line
# Verify presence of named file
tarFile="$2"
tarFile=${tarFile##*/}
if [ ! -e $tarFile ]; then
echo "The file $tarFile does not exist." exit 4
fi
# Verify parameter is really a tar file
isTar "$tarFile"
if [[ ! $? -eq 0 ]]; then
echo "The file $tarFile is not a valid tar file."
exit 5
fi
############################################################ Get student's PID
# Extract first token of tar file name (student PID when we run this)
getPID $tarFile
# Initiate header for grading log
headerLog="header.txt"
echo "Grading: $2" > $headerLog
echo -n "Time: " >> $headerLog
echo `date` >> $headerLog
echo "Option: $1" >> $headerLog
echo >> $headerLog
################################################### Clean up the test directory
# Remove pre-existing assembler executable from test directory
if [[ -e assemble ]]; then
rm -f assemble
fi
# Remove any old student object files and assembler executable
rm -f ./*.o
rm -f ./$exeName
############################################################ Prepare for build
# Create log file:
echo "Executing grade.sh..." > $buildLog
echo >> $buildLog
# Create build directory:
echo "Creating build subdirectory" >> $buildLog
echo >> $buildLog
# Create build directory if needed; empty it if it already exists
if [[ -d $buildDir ]]; then
rm -Rf ./$buildDir/*
else
mkdir $buildDir
fi
# Extract student's tar file to the build directory
echo "Extracting student's files to the build directory:" >> $buildLog
tar xvf $tarFile -C ./$buildDir >> $buildLog
echo >> $buildLog
# Move to build directory
cd ./build
# Verify we have a makefile
gotMakefile="yes"
if [[ ! -e makefile ]] && [[ ! -e Makefile ]] && [[ ! -e GNUmakefile ]]; then
echo "There is no makefile in the build directory" >> ../$buildLog
gotMakefile="no"
#echo $Separator >> ../$buildLog
#mv ../$buildLog ../$spid.txt
#exit 6
fi
# Verify we have a pledge file
if [[ ! -e pledge.txt ]]; then
echo "Missing pledge file" >> ../$buildLog
fi
# Remove extraneous files from build directory
if [[ -e assemble ]]; then
echo "Deleting prebuilt assembler" >> ../$buildLog
rm -f assemble
fi
if [[ -e *.o ]]; then
echo "Deleting extraneous object files" >> ../$buildLog
rm -f *.o
fi
############################################################ Build the assembler
# build the assembler; save output in build log
if [[ $gotMakefile == "yes" ]]; then
echo "Invoking: make $makeTarget" >> ../$buildLog
make $makeTarget >> ../$buildLog 2>&1
echo >> ../$buildLog
if [[ ! -e $exeName ]]; then
echo "$exeName was not found... trying make with default target..." >> ../$buildLog
# try to make a default target
make
if [[ ! -e $exeName ]]; then
echo "That failed; the file $exeName does not exist" >> ../$buildLog
echo $Separator >> ../$buildLog
else
echo "Attempting to build w/o a makefile; penalty will be 20%." >> ../$buildLog
gcc -o $exeName -std=c99 -Wall -ggdb3 *.c >> ../$buildLog 2>&1
fi
fi
else
echo "ERROR: no makefile found! " >> ../$buildLog
echo "Attempting to build w/o a makefile; penalty will be 20%." >> ../$buildLog
gcc -o $exeName -std=c11 -Wall -ggdb3 *.c >> ../$buildLog 2>&1
fi
# Verify existence of executable
if [[ ! -e $exeName ]]; then
echo "I give up... cannot build $exeName..." >> ../$buildLog
mv ../$buildLog ../$spid.txt
exit 7
fi
if [[ ! -x assemble ]]; then
echo "Permissions error; the file $exeName is not executable" >> ../$buildLog
echo $Separator >> ../$buildLog
mv ../$buildLog ../$spid.txt
exit 8
fi
echo "Build succeeded..." >> $buildLog
# Check for 32-bit executable and issue warning
file ./$exeName | grep "32-bit"
if [[ $? -eq 0 ]]; then
warningLog="warningLog.txt"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" >> $warningLog
echo "Warning: $exeName apears to be a 32-bit executable." >> $warningLog
echo "This will interfere with the valgrind checks, and affect your score." >> $warningLog
echo "We strongly suggest you remove -m32 from your makefile." >> $warningLog
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" >> $warningLog
echo >> $warningLog
cat $warningLog >> ../$headerLog
echo
cat $warningLog
rm -f $warningLog
fi
# Move executable up to test directory and return there
echo "Moving the executable $exeName to the test directory." >> $buildLog
mv ./$exeName .. >> $buildLog
cd .. >> $buildLog
# Delimit this section of the report file:
echo $Separator >> $buildLog
############################################################ Perform testing
# Initiate test Log
echo "Begin running test cases..." > $testLog
echo >> $testLog
# perform testing
index=0
############################## run the assemble test cases
while [ "$index" -lt "$nCases" ]
do
asmToProcess=$testDataDir/${asmFileNames[$index]}
refObjectFile=$testDataDir/${refFileNames[$index]}
stuObjectFile=${stuFileNames[$index]}
((index +=1))
echo "Beginning test case $index" >> $testLog
# verify existence of current test file and reference file
if [ ! -e $asmToProcess ]; then
echo "The test file $asmToProcess is not present" >> $testLog
continue
fi
if [ ! -e $refObjectFile ]; then
echo "The reference file $refObjectFile is not present" >> $testLog
continue
fi
# write-protect test files
chmod a-w $asmToProcess
chmod a-w $refObjectFile
# run assembler on current test file
echo "Running assembler on $asmToProcess" >> $testLog
timeout -s SIGKILL 30 ./$exeName $asmToProcess $stuObjectFile >> $testLog
if [[ $timeoutStatus -eq 124 || $timeoutStatus -eq 137 ]]; then
echo "The test of your solution timed out after 30 seconds." >> $testLog
echo "Valgrind testing will NOT be done." >> $testLog
killed="yes"
elif [[ $timeoutStatus -eq 134 ]]; then
echo "The test of your solution was killed by a SIGABRT signal." >> $testLog
echo "Possible reasons include:" >> $testLog
echo " - a segfault error" >> $testLog
echo " - a serious memory access error" >> $testLog
echo "Valgrind testing will NOT be done." >> $testLog
killed="yes"
fi
# remove write-protection from test files
chmod u+w $asmToProcess
chmod u+w $refObjectFile
# verify existence of a nonempty student object file
if [ ! -s $stuObjectFile ]; then
echo "The object file $stuObjectFile was not produced or is empty" >> $testLog
echo >> $testLog
continue
fi
# run comparator
echo "Running compare on $stuObjectFile and $refObjectFile" >> $testLog
./compare $index $stuObjectFile $refObjectFile >> $testLog
echo >> $testLog
done
############################################################ Run valgrind check
if [[ $1 == "-final" ]]; then
#echo "Running valgrind test"
# run test 7 on valgrind
# full valgrind output is in $vgrindLog
# extracted counts are in $vgrindIssues
vgrindLog="vgrindLog.txt"
echo "Running valgrind test..." >> $vgrindLog
vgrindSwitches=" --leak-check=full --show-leak-kinds=all --log-file=$vgrindLog --track-origins=yes -v"
scoreResultsLog2="ScoresValgrind.txt"
timeout -s SIGKILL 30 valgrind $vgrindSwitches ./assemble ./final/ftest07.asm ./fstu07.o
if [[ $timeoutStatus -eq 124 || $timeoutStatus -eq 137 ]]; then
echo "The test of your solution timed out after 30 seconds." >> $testLog
echo "Valgrind testing will NOT be done." >> $testLog
killed="yes"
elif [[ $timeoutStatus -eq 134 ]]; then
echo "The test of your solution was killed by a SIGABRT signal." >> $testLog
echo "Possible reasons include:" >> $testLog
echo " - a segfault error" >> $testLog
echo " - a serious memory access error" >> $testLog
echo "Valgrind testing will NOT be done." >> $testLog
killed="yes"
fi
# accumulate valgrind error counts
if [[ -e $vgrindLog ]]; then
vgrindIssues="vgrind_issues.txt"
echo "Valgrind issues:" > $vgrindIssues
grep "in use at exit" $vgrindLog >> $vgrindIssues
grep "total heap usage" $vgrindLog >> $vgrindIssues
grep "definitely lost" $vgrindLog >> $vgrindIssues
echo "Invalid reads: `grep -c "Invalid read" $vgrindLog`" >> $vgrindIssues
echo "Invalid writes: `grep -c "Invalid write" $vgrindLog`" >> $vgrindIssues
echo "Uses of uninitialized values: `grep -c "uninitialised" $vgrindLog`" >> $vgrindIssues
else
echo "Error running Valgrind test." >> $testLog
fi
fi
############################################################ Run symbol table tests
if [[ $1 == "-final" ]]; then
symbolsLog="symbolsLog.txt"
echo "Details from symbol table generation tests:" > $symbolsLog
echo >> $symbolsLog
timeout -s SIGKILL 30 ./assemble ./final/ftest01.asm ./symstu01.txt -symbols
if [[ $timeoutStatus -eq 124 || $timeoutStatus -eq 137 ]]; then
echo "The test of your solution timed out after 30 seconds." >> $testLog
echo "Valgrind testing will NOT be done." >> $testLog
killed="yes"
elif [[ $timeoutStatus -eq 134 ]]; then
echo "The test of your solution was killed by a SIGABRT signal." >> $testLog
echo "Possible reasons include:" >> $testLog
echo " - a segfault error" >> $testLog
echo " - a serious memory access error" >> $testLog
echo "Valgrind testing will NOT be done." >> $testLog
killed="yes"
fi
if [[ -e "./symstu01.txt" ]]; then
sort -k2 ./symstu01.txt > sortedsyms01.txt
echo "Sorted symbol table for ftest01.asm:" >> $symbolsLog
cat sortedsyms01.txt >> $symbolsLog
echo >> $symbolsLog
./symcompare 11 sortedsyms01.txt ./symtabs/refsyms01.txt >> $symbolsLog
echo >> $symbolsLog
else
echo " File not found: symstu01.txt" >> $symbolsLog
fi
timeout -s SIGKILL 30 ./assemble ./final/ftest04.asm ./symstu04.txt -symbols
if [[ $timeoutStatus -eq 124 || $timeoutStatus -eq 137 ]]; then
echo "The test of your solution timed out after 30 seconds." >> $testLog
echo "Valgrind testing will NOT be done." >> $testLog
killed="yes"
elif [[ $timeoutStatus -eq 134 ]]; then
echo "The test of your solution was killed by a SIGABRT signal." >> $testLog
echo "Possible reasons include:" >> $testLog
echo " - a segfault error" >> $testLog
echo " - a serious memory access error" >> $testLog
echo "Valgrind testing will NOT be done." >> $testLog
killed="yes"
fi
if [[ -e "./symstu04.txt" ]]; then
sort -k2 ./symstu04.txt > sortedsyms04.txt
echo "Sorted symbol table for ftest04.asm:" >> $symbolsLog
cat sortedsyms04.txt >> $symbolsLog
echo >> $symbolsLog
./symcompare 12 sortedsyms04.txt ./symtabs/refsyms04.txt >> $symbolsLog
echo >> $symbolsLog
else
echo " File not found: symstu04.txt" >> $symbolsLog
fi
timeout -s SIGKILL 30 ./assemble ./final/ftest07.asm ./symstu07.txt -symbols
if [[ $timeoutStatus -eq 124 || $timeoutStatus -eq 137 ]]; then
echo "The test of your solution timed out after 30 seconds." >> $testLog
echo "Valgrind testing will NOT be done." >> $testLog
killed="yes"
elif [[ $timeoutStatus -eq 134 ]]; then
echo "The test of your solution was killed by a SIGABRT signal." >> $testLog
echo "Possible reasons include:" >> $testLog
echo " - a segfault error" >> $testLog
echo " - a serious memory access error" >> $testLog
echo "Valgrind testing will NOT be done." >> $testLog
killed="yes"
fi
if [[ -e "./symstu07.txt" ]]; then
sort -k2 ./symstu07.txt > sortedsyms07.txt
echo "Sorted symbol table for ftest07.asm:" >> $symbolsLog
cat sortedsyms07.txt >> $symbolsLog
echo >> $symbolsLog
./symcompare 13 sortedsyms07.txt ./symtabs/refsyms07.txt >> $symbolsLog
echo $Separator >> $summaryLog
else
echo " File not found: symstu07.txt" >> $symbolsLog
fi
echo >> $symbolsLog
fi
############################################################ File report
# complete summary file
# write header to summary log
cat "$headerLog" > $summaryLog
echo ">>Scores from testing<<" >> $summaryLog
# write scores to summary file
#echo >> $summaryLog
grep Score $testLog >> $summaryLog
if [[ $1 == "-final" ]]; then
grep symbols $symbolsLog >> $summaryLog
fi
echo $Separator >> $summaryLog
echo >> $summaryLog
# write Valgrind summary into log
if [[ $1 == "-final" ]]; then
cat $vgrindIssues >> $summaryLog
echo $Separator >> $summaryLog
fi
# write symbol tables test results into log
if [[ $1 == "-final" ]]; then
cat $symbolsLog >> $summaryLog
echo $Separator >> $summaryLog
fi
# write testing details into summary
cat $testLog >> $summaryLog
echo $Separator >> $summaryLog
# write Valgrind details into log
if [[ $1 == "-final" ]]; then
cat $vgrindLog >> $summaryLog
echo $Separator >> $summaryLog
fi
# write build log into summary
echo "Results from $buildLog" >> $summaryLog
cat $buildLog >> $summaryLog
echo >> $summaryLog
# rename summary log using student's PID
mv $summaryLog $spid.txt
exit 0