run_tests.sh 7.04 KB
Newer Older
1 2 3
#!/bin/bash

IP=../src/interpret
4
[ "$OS" == "Windows_NT" ] && IP=../src/interpret.exe
5 6 7 8

RED="\033[0;31m"
GREEN="\033[0;32m"
YELLOW="\033[0;33m"
Camil Staps's avatar
Camil Staps committed
9
PURPLE="\033[0;35m"
10 11
RESET="\033[0m"

Camil Staps's avatar
Camil Staps committed
12
FAILED=()
13

Camil Staps's avatar
Camil Staps committed
14
RUNFLAGS=""
Camil Staps's avatar
Camil Staps committed
15
NATIVE_RUNFLAGS=""
16

17
WASM=0
18 19
INTERPRETERGENWASMFLAGS=""

20
SRCMAKETARGETS="all"
21
MAKE=1
Camil Staps's avatar
Camil Staps committed
22
BENCHMARK=0
23
EXPECTED_PREFIX=".64"
24
BC_EXTENSION="bc"
25
RUN_ONLY=()
26
PROFILE=0
27
QUIET=0
Camil Staps's avatar
Camil Staps committed
28
OPTIMISE=1
29
JUNIT_EXPORT=0
Camil Staps's avatar
Camil Staps committed
30 31 32 33 34 35 36

cpprj () {
	if [ $OPTIMISE -gt 0 ]; then
		sed 's/OptimiseABC:.*/OptimiseABC:\tTrue/' "$1" > "$2"
	else
		sed 's/OptimiseABC:.*/OptimiseABC:\tFalse/' "$1" > "$2"
	fi
37
	[ "$OS" == "Windows_NT" ] && sed -i 's:\*lib\*:*Libraries*:' "$2"
Camil Staps's avatar
Camil Staps committed
38 39
}

40
cpmq () {
41
	res="$(cpm $@)"
42
	ecode=$?
43 44 45 46 47 48 49 50
	echo "$res" | grep -i 'Error' >/dev/null
	if [ $ecode -ne 0 ]; then
		echo "$res" | grep -v Analyzing | grep -i '^\|Error\|Warning'
		return -1
	else
		echo "$res" | grep --color=never -i 'Finished making.'
		return 0
	fi
Camil Staps's avatar
Camil Staps committed
51
}
52

53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76
junit_export () {
	MODULE="$1"
	RESULT="$2"
	EXPECTED_FILE="$3"
	FAILURES=0
	if [ "$RESULT" = "failed" ]; then
		FAILURES=1
	fi

	echo "<?xml version=\"1.0\"?>"
	echo "<testsuites tests=\"1\" failures=\"$FAILURES\" time=\"0\">"
	echo "<testsuite name=\"$MODULE\" tests=\"1\" failures=\"$FAILURES\" time=\"0\">"
	echo "<testcase id=\"$MODULE\" name=\"$MODULE\" classname=\"$MODULE\" time=\"0\">"
	if [ "$RESULT" = "failed" ]; then
		echo "<failure>"
		git diff --no-index --word-diff -U0 $EXPECTED $MODULE.result \
			| sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g'
		echo "</failure>"
	fi
	echo "</testcase>"
	echo "</testsuite>"
	echo "</testsuites>"
}

77 78 79 80
print_help () {
	echo "$0: run tests"
	echo
	echo "Options:"
81
	echo "  -H       Print this help"
82
	echo
83
	echo "  -M       Don't make ../src before running tests"
84
	echo "  -o TEST  Only run test TEST"
85 86
	echo "  -q       Don't show program results"
	echo "  -b       Run benchmarks"
87
	echo
88
	echo "  -w       Use the WebAssembly interpreter (does not support all options below)"
89
	echo "  -3       Run tests as if on a 32-bit machine"
90 91
	echo "  -f       Compile the interpreter with -Ofast -fno-unsafe-math-optimizations"
	echo "  -O       Skip the ABC optimisation step"
92 93
	echo
	echo "  -h SIZE  Set heap size to SIZE"
94
	echo "  -s SIZE  Set stack size to SIZE"
95
	echo
96 97 98
	echo "  -d       Print all instructions as they are executed"
	echo "  -l       List bytecode before execution"
	echo "  -p       Make PDF profiles (e.g. nfib.prof.pdf) using google-pprof"
99
	echo "  -j       Output JUnit style XML files (for GitLab CI)"
100 101 102 103
	exit 0
}

print_usage () {
104
	echo "Usage: $0 OPTS (see -H for details)"
105 106 107
	exit 1
}

108 109 110 111 112 113 114
contains () {
	local e match="$1"
	shift
	for e; do [[ "$e" == "$match" ]] && return 0; done
	return 1
}

115
OPTS=`getopt "Ho:wbMfh:Os:3dlpjq" "$@"` || print_usage
116 117 118 119
eval set -- "$OPTS"

while true; do
	case "$1" in
120
		-H)
121
			print_help;;
122

123
		-o)
124
			RUN_ONLY+=("$2")
125
			shift 2;;
126

127 128
		-w)
			WASM=1
129
			IP="js --test-wasm-await-tier2 ../src-js/interpret.js"
Camil Staps's avatar
Camil Staps committed
130
			SRCMAKETARGETS="abcopt bcgen bclink bcprelink bcstrip"
131
			BC_EXTENSION="pbc"
132 133
			shift;;

134
		-b)
Camil Staps's avatar
Camil Staps committed
135 136
			BENCHMARK=1
			shift;;
137
		-f)
138
			SRCMAKETARGETS+=" optimized"
139
			shift;;
140 141 142
		-M)
			MAKE=0
			shift;;
143
		-h)
144
			RUNFLAGS+=" -h $2"
Camil Staps's avatar
Camil Staps committed
145
			NATIVE_RUNFLAGS+=" -h $2"
146
			shift 2;;
147
		-O)
Camil Staps's avatar
Camil Staps committed
148
			OPTIMISE=0
149
			shift;;
150
		-s)
151
			RUNFLAGS+=" -s $2"
Camil Staps's avatar
Camil Staps committed
152
			NATIVE_RUNFLAGS+=" -s $2"
153
			shift 2;;
154
		-3)
155
			EXPECTED_PREFIX=".32"
156 157
			CFLAGS+=" -m32 -DWORD_WIDTH=32"
			shift;;
158

159
		-d)
160
			CFLAGS+=" -DDEBUG_ALL_INSTRUCTIONS"
161
			INTERPRETERGENWASMFLAGS+=" -d"
162
			shift;;
163
		-l)
164
			RUNFLAGS+=" -l"
165
			shift;;
166
		-p)
167 168 169 170 171
			CFLAGS+=" -g -lprofiler"
			export CPUPROFILE=/tmp/prof.out
			export CPUPROFILE_FREQUENCY=10000
			PROFILE=1
			shift;;
172 173 174
		-j)
			JUNIT_EXPORT=1
			shift;;
175
		-q)
176 177
			QUIET=1
			shift;;
178

179 180 181
		--)
			shift
			break;;
182
		*)
183
			break;;
184 185 186
	esac
done

187
if [ $BENCHMARK -gt 0 ] && [ $WASM -eq 0 ] && [[ $SRCMAKETARGETS != *"optimized"* ]]; then
188 189 190 191
	echo -e "${RED}Warning: benchmarking without compiler optimisations (did you forget -f?)$RESET"
	sleep 1
fi

192
if [ "$OS" != "Windows_NT" ] && [ $MAKE -gt 0 ] ; then
193 194 195 196
	CFLAGS="$CFLAGS" make -BC ../src $SRCMAKETARGETS || exit 1
fi

if [ $WASM -gt 0 ]; then
197
	INTERPRETERGENWASMFLAGS="$INTERPRETERGENWASMFLAGS" make -BC ../src-js all || exit 1
198 199 200 201

	if [ $BENCHMARK -gt 0 ]; then
		RUNFLAGS+=" --time"
	fi
202
fi
203

Camil Staps's avatar
Camil Staps committed
204
for MODULE in *.icl
205
do
Camil Staps's avatar
Camil Staps committed
206
	MODULE="${MODULE/.icl/}"
Camil Staps's avatar
Camil Staps committed
207

208
	if [[ "$MODULE" == "CodeSharing" ]] || [[ "$MODULE" == "GraphTest" ]]; then
Camil Staps's avatar
Camil Staps committed
209
		continue
210 211
	elif [[ "$MODULE" == "long_integers" ]] && [ $WASM -gt 0 ]; then
		continue
212
	elif [[ "${#RUN_ONLY[@]}" -gt 0 ]] && ! contains "$MODULE" "${RUN_ONLY[@]}"; then
213
		continue
Camil Staps's avatar
Camil Staps committed
214 215
	fi

216
	if [ $BENCHMARK -gt 0 ]; then
217
		cp "$MODULE.icl" "$MODULE.icl.nobm"
218
		if [ -f "$MODULE.bm.sed" ]; then
219
			sed -i.nobm -f "$MODULE.bm.sed" "$MODULE.icl"
220
		fi
Camil Staps's avatar
Camil Staps committed
221 222
	fi

Camil Staps's avatar
Camil Staps committed
223 224
	echo -e "${YELLOW}Running $MODULE...$RESET"

Camil Staps's avatar
Camil Staps committed
225 226 227
	cpprj "$MODULE.prj.default" "$MODULE.prj"
	cpmq project "$MODULE.prj" build
	CPMRESULT=$?
Camil Staps's avatar
Camil Staps committed
228

Camil Staps's avatar
Camil Staps committed
229
	if [ $CPMRESULT -ne 0 ]; then
230
		echo -e "${RED}FAILED: $MODULE (compilation)$RESET"
231
		[ $BENCHMARK -gt 0 ] && mv "$MODULE.icl.nobm" "$MODULE.icl"
Camil Staps's avatar
Camil Staps committed
232
		FAILED+=("$MODULE")
Camil Staps's avatar
Camil Staps committed
233
		continue
Camil Staps's avatar
Camil Staps committed
234 235
	fi

236 237
	if [ ! -f "$MODULE$EXPECTED_PREFIX.expected" ]; then
		echo -e "${YELLOW}Skipping $MODULE (no expected outcome)${RESET}"
238
		[ $BENCHMARK -gt 0 ] && mv "$MODULE.icl.nobm" "$MODULE.icl"
Camil Staps's avatar
Camil Staps committed
239
		continue
240
	fi
Camil Staps's avatar
Camil Staps committed
241

242 243
	MODULE_HEAPSIZE="$(grep -w HeapSize "$MODULE.prj" | cut -f4 | tr -d '\r')"
	MODULE_STACKSIZE="$(grep -w StackSize "$MODULE.prj" | cut -f4 | tr -d '\r')"
Camil Staps's avatar
Camil Staps committed
244 245
	MODULE_RUNFLAGS="-h $MODULE_HEAPSIZE -s $MODULE_STACKSIZE"

246
	[ $BENCHMARK -gt 0 ] && mv "$MODULE.icl.nobm" "$MODULE.icl"
Camil Staps's avatar
Camil Staps committed
247

248
	if [ $BENCHMARK -gt 0 ]; then
249
		/usr/bin/time -p $IP $MODULE_RUNFLAGS $RUNFLAGS $MODULE.$BC_EXTENSION 2>bm-tmp >$MODULE.result
250
		WALL_TIME="$(grep user bm-tmp | sed 's/user[[:space:]]*//' | head -n 1)"
251 252 253
		/usr/bin/time -p ./"$MODULE" $MODULE_RUNFLAGS $NATIVE_RUNFLAGS -nt -nr 2>bm-tmp
		WALL_TIME_NATIVE="$(grep user bm-tmp | sed 's/user[[:space:]]*//')"
		rm bm-tmp
254 255 256 257 258 259
		if [ "$WALL_TIME_NATIVE" == "0.00" ]; then
			WALL_TIME_RATIO="ratio not computable"
		else
			WALL_TIME_RATIO="$(echo "scale=3;$WALL_TIME/$WALL_TIME_NATIVE" | bc)x"
		fi
		echo -e "${PURPLE}Time used: $WALL_TIME / $WALL_TIME_NATIVE (${WALL_TIME_RATIO})$RESET"
Camil Staps's avatar
Camil Staps committed
260
	elif [ $QUIET -gt 0 ]; then
261
		/usr/bin/time $IP $MODULE_RUNFLAGS $RUNFLAGS $MODULE.$BC_EXTENSION > $MODULE.result
262
	else
263
		/usr/bin/time $IP $MODULE_RUNFLAGS $RUNFLAGS $MODULE.$BC_EXTENSION | tee $MODULE.result
264
	fi
Camil Staps's avatar
Camil Staps committed
265

266
	if [ $PROFILE -ne 0 ]; then
267
		google-pprof --pdf $IP /tmp/prof.out > $MODULE.prof.pdf
268 269
	fi

270 271
	[ "$OS" == "Windows_NT" ] && dos2unix $MODULE.result

272
	EXPECTED="$MODULE$EXPECTED_PREFIX.expected"
273
	if [ $BENCHMARK -gt 0 ] && [ -f "$MODULE.bm$EXPECTED_PREFIX.expected" ]; then
274
		EXPECTED="$MODULE.bm$EXPECTED_PREFIX.expected"
275
	fi
276
	git diff --no-index --word-diff -U0 $EXPECTED $MODULE.result
Camil Staps's avatar
Camil Staps committed
277
	if [ $? -ne 0 ]; then
278
		[ $JUNIT_EXPORT -gt 0 ] && junit_export $MODULE failed "$EXPECTED" > "$MODULE.junit.xml"
279
		echo -e "${RED}FAILED: $MODULE (different result)$RESET"
Camil Staps's avatar
Camil Staps committed
280
		FAILED+=("$MODULE")
281
	else
282
		[ $JUNIT_EXPORT -gt 0 ] && junit_export "$MODULE" passed "$EXPECTED" > "$MODULE.junit.xml"
283
		echo -e "${GREEN}Passed: $MODULE$RESET"
Camil Staps's avatar
Camil Staps committed
284
	fi
Camil Staps's avatar
Camil Staps committed
285
done
286

Camil Staps's avatar
Camil Staps committed
287
if [ ${#FAILED[@]} -eq 0 ]; then
Camil Staps's avatar
Camil Staps committed
288
	echo -e "${GREEN}All tests passed$RESET"
Camil Staps's avatar
Camil Staps committed
289
else
Camil Staps's avatar
Camil Staps committed
290
	echo -e "${RED}Some tests failed: ${FAILED[@]}$RESET"
Camil Staps's avatar
Camil Staps committed
291
	exit 1
Camil Staps's avatar
Camil Staps committed
292
fi