run_tests.sh 6.35 KB
Newer Older
1 2 3
#!/bin/bash

IP=../src/interpret
4
[ "$OS" == "Windows_NT" ] && IP=../src/interpret.exe
5 6 7 8

RED="\033[0;31m"
GREEN="\033[0;32m"
YELLOW="\033[0;33m"
Camil Staps's avatar
Camil Staps committed
9
PURPLE="\033[0;35m"
10 11
RESET="\033[0m"

Camil Staps's avatar
Camil Staps committed
12
FAILED=()
13

Camil Staps's avatar
Camil Staps committed
14
RUNFLAGS=""
Camil Staps's avatar
Camil Staps committed
15
NATIVE_RUNFLAGS=""
16

17
WASM=0
18 19
INTERPRETERGENWASMFLAGS=""

20
SRCMAKETARGETS="all"
Camil Staps's avatar
Camil Staps committed
21
BENCHMARK=0
22
EXPECTED_PREFIX=".64"
23
BC_EXTENSION="bc"
24
RUN_ONLY=()
25
PROFILE=0
26
RECOMPILE=1
27
QUIET=0
Camil Staps's avatar
Camil Staps committed
28 29 30 31 32 33 34 35
OPTIMISE=1

cpprj () {
	if [ $OPTIMISE -gt 0 ]; then
		sed 's/OptimiseABC:.*/OptimiseABC:\tTrue/' "$1" > "$2"
	else
		sed 's/OptimiseABC:.*/OptimiseABC:\tFalse/' "$1" > "$2"
	fi
36
	[ "$OS" == "Windows_NT" ] && sed -i 's:\*lib\*:*Libraries*:' "$2"
Camil Staps's avatar
Camil Staps committed
37 38 39
}

cpmq() {
40
	res="$(cpm $@)"
41
	ecode=$?
42 43 44 45 46 47 48 49
	echo "$res" | grep -i 'Error' >/dev/null
	if [ $ecode -ne 0 ]; then
		echo "$res" | grep -v Analyzing | grep -i '^\|Error\|Warning'
		return -1
	else
		echo "$res" | grep --color=never -i 'Finished making.'
		return 0
	fi
Camil Staps's avatar
Camil Staps committed
50
}
51

52 53 54 55
print_help () {
	echo "$0: run tests"
	echo
	echo "Options:"
56
	echo "  -H       Print this help"
57
	echo
58
	echo "  -o TEST  Only run test TEST"
59
	echo
60 61
	echo "  -w       Use the WebAssembly interpreter (does not support all options below)"
	echo
62 63 64 65 66 67 68
	echo "  -b       Run benchmarks"
	echo "  -f       Compile the interpreter with -Ofast -fno-unsafe-math-optimizations"
	echo "  -h SIZE  Set heap size to SIZE"
	echo "  -O       Skip the ABC optimisation step"
	echo "  -s SIZE  Set stack size to SIZE"
	echo "  -3       Run tests as if on a 32-bit machine"
	echo "  -R       Don't recompile modules (faster, but halt addresses may be incorrect if optimisations are missed)"
69
	echo
70 71 72 73
	echo "  -d       Print all instructions as they are executed"
	echo "  -l       List bytecode before execution"
	echo "  -p       Make PDF profiles (e.g. nfib.prof.pdf) using google-pprof"
	echo "  -q       Don't show program results"
74 75 76 77
	exit 0
}

print_usage () {
78
	echo "Usage: $0 OPTS (see -H for details)"
79 80 81
	exit 1
}

82 83 84 85 86 87 88
contains () {
	local e match="$1"
	shift
	for e; do [[ "$e" == "$match" ]] && return 0; done
	return 1
}

89
OPTS=`getopt "Ho:wbfh:Os:3Rdlpq" "$@"` || print_usage
90 91 92 93
eval set -- "$OPTS"

while true; do
	case "$1" in
94
		-H)
95
			print_help;;
96

97
		-o)
98
			RUN_ONLY+=("$2")
99
			shift 2;;
100

101 102 103
		-w)
			WASM=1
			IP="js ../src-js/interpret.js"
Camil Staps's avatar
Camil Staps committed
104
			SRCMAKETARGETS="abcopt bcgen bclink bcprelink bcstrip"
105 106 107
			BC_EXTENSION="ubc"
			shift;;

108
		-b)
Camil Staps's avatar
Camil Staps committed
109 110
			BENCHMARK=1
			shift;;
111
		-f)
112
			SRCMAKETARGETS+=" optimized"
113
			shift;;
114
		-h)
115
			RUNFLAGS+=" -h $2"
Camil Staps's avatar
Camil Staps committed
116
			NATIVE_RUNFLAGS+=" -h $2"
117
			shift 2;;
118
		-O)
Camil Staps's avatar
Camil Staps committed
119
			OPTIMISE=0
120
			shift;;
121
		-s)
122
			RUNFLAGS+=" -s $2"
Camil Staps's avatar
Camil Staps committed
123
			NATIVE_RUNFLAGS+=" -s $2"
124
			shift 2;;
125
		-3)
126
			EXPECTED_PREFIX=".32"
127 128
			CFLAGS+=" -m32 -DWORD_WIDTH=32"
			shift;;
129
		-R)
130 131
			RECOMPILE=0
			shift;;
132

133
		-d)
134
			CFLAGS+=" -DDEBUG_ALL_INSTRUCTIONS"
135
			INTERPRETERGENWASMFLAGS+=" -d"
136
			shift;;
137
		-l)
138
			RUNFLAGS+=" -l"
139
			shift;;
140
		-p)
141 142 143 144 145
			CFLAGS+=" -g -lprofiler"
			export CPUPROFILE=/tmp/prof.out
			export CPUPROFILE_FREQUENCY=10000
			PROFILE=1
			shift;;
146
		-q)
147 148
			QUIET=1
			shift;;
149

150 151 152
		--)
			shift
			break;;
153
		*)
154
			break;;
155 156 157
	esac
done

158
if [ $BENCHMARK -gt 0 ] && [ $WASM -eq 0 ] && [[ $SRCMAKETARGETS != *"optimized"* ]]; then
159 160 161 162
	echo -e "${RED}Warning: benchmarking without compiler optimisations (did you forget -f?)$RESET"
	sleep 1
fi

163
if [ "$OS" != "Windows_NT" ]; then
164 165 166 167
	CFLAGS="$CFLAGS" make -BC ../src $SRCMAKETARGETS || exit 1
fi

if [ $WASM -gt 0 ]; then
168
	INTERPRETERGENWASMFLAGS="$INTERPRETERGENWASMFLAGS" make -BC ../src-js all || exit 1
169 170 171 172

	if [ $BENCHMARK -gt 0 ]; then
		RUNFLAGS+=" --time"
	fi
173
fi
174

175
if [ $RECOMPILE -gt 0 ]; then
176
	rm -fr Clean\ System\ Files
177
fi
Camil Staps's avatar
Camil Staps committed
178

Camil Staps's avatar
Camil Staps committed
179
for MODULE in *.icl
180
do
Camil Staps's avatar
Camil Staps committed
181
	MODULE="${MODULE/.icl/}"
Camil Staps's avatar
Camil Staps committed
182

183
	if [[ "$MODULE" == "CodeSharing" ]] || [[ "$MODULE" == "GraphTest" ]]; then
Camil Staps's avatar
Camil Staps committed
184
		continue
185 186
	elif [[ "$MODULE" == "long_integers" ]] && [ $WASM -gt 0 ]; then
		continue
187
	elif [[ "${#RUN_ONLY[@]}" -gt 0 ]] && ! contains "$MODULE" "${RUN_ONLY[@]}"; then
188
		continue
Camil Staps's avatar
Camil Staps committed
189 190
	fi

191
	if [ $BENCHMARK -gt 0 ]; then
192
		cp "$MODULE.icl" "$MODULE.icl.nobm"
193
		if [ -f "$MODULE.bm.sed" ]; then
194
			sed -i.nobm -f "$MODULE.bm.sed" "$MODULE.icl"
195
		fi
Camil Staps's avatar
Camil Staps committed
196 197
	fi

Camil Staps's avatar
Camil Staps committed
198 199
	echo -e "${YELLOW}Running $MODULE...$RESET"

Camil Staps's avatar
Camil Staps committed
200 201 202
	cpprj "$MODULE.prj.default" "$MODULE.prj"
	cpmq project "$MODULE.prj" build
	CPMRESULT=$?
Camil Staps's avatar
Camil Staps committed
203

Camil Staps's avatar
Camil Staps committed
204
	if [ $CPMRESULT -ne 0 ]; then
205
		echo -e "${RED}FAILED: $MODULE (compilation)$RESET"
206
		[ $BENCHMARK -gt 0 ] && mv "$MODULE.icl.nobm" "$MODULE.icl"
Camil Staps's avatar
Camil Staps committed
207
		FAILED+=("$MODULE")
Camil Staps's avatar
Camil Staps committed
208 209
		continue
	fi
210
	if [ $RECOMPILE -gt 0 ]; then
Camil Staps's avatar
Camil Staps committed
211 212 213
		cpmq project "$MODULE.prj" build
	fi

214 215
	if [ ! -f "$MODULE$EXPECTED_PREFIX.expected" ]; then
		echo -e "${YELLOW}Skipping $MODULE (no expected outcome)${RESET}"
216
		[ $BENCHMARK -gt 0 ] && mv "$MODULE.icl.nobm" "$MODULE.icl"
Camil Staps's avatar
Camil Staps committed
217
		continue
218
	fi
Camil Staps's avatar
Camil Staps committed
219

220 221
	MODULE_HEAPSIZE="$(grep -w HeapSize "$MODULE.prj" | cut -f4 | tr -d '\r')"
	MODULE_STACKSIZE="$(grep -w StackSize "$MODULE.prj" | cut -f4 | tr -d '\r')"
Camil Staps's avatar
Camil Staps committed
222 223
	MODULE_RUNFLAGS="-h $MODULE_HEAPSIZE -s $MODULE_STACKSIZE"

224
	[ $BENCHMARK -gt 0 ] && mv "$MODULE.icl.nobm" "$MODULE.icl"
Camil Staps's avatar
Camil Staps committed
225

Camil Staps's avatar
Camil Staps committed
226
	[ $WASM -gt 0 ] && ../src/bcprelink "$MODULE.bc" -o "$MODULE.ubc"
227

228
	if [ $BENCHMARK -gt 0 ]; then
229
		/usr/bin/time -p $IP $MODULE_RUNFLAGS $RUNFLAGS $MODULE.$BC_EXTENSION 2>bm-tmp >$MODULE.result
230
		WALL_TIME="$(grep user bm-tmp | sed 's/user[[:space:]]*//' | head -n 1)"
231 232 233
		/usr/bin/time -p ./"$MODULE" $MODULE_RUNFLAGS $NATIVE_RUNFLAGS -nt -nr 2>bm-tmp
		WALL_TIME_NATIVE="$(grep user bm-tmp | sed 's/user[[:space:]]*//')"
		rm bm-tmp
234 235 236 237 238 239
		if [ "$WALL_TIME_NATIVE" == "0.00" ]; then
			WALL_TIME_RATIO="ratio not computable"
		else
			WALL_TIME_RATIO="$(echo "scale=3;$WALL_TIME/$WALL_TIME_NATIVE" | bc)x"
		fi
		echo -e "${PURPLE}Time used: $WALL_TIME / $WALL_TIME_NATIVE (${WALL_TIME_RATIO})$RESET"
Camil Staps's avatar
Camil Staps committed
240
	elif [ $QUIET -gt 0 ]; then
241
		/usr/bin/time $IP $MODULE_RUNFLAGS $RUNFLAGS $MODULE.$BC_EXTENSION > $MODULE.result
242
	else
243
		/usr/bin/time $IP $MODULE_RUNFLAGS $RUNFLAGS $MODULE.$BC_EXTENSION | tee $MODULE.result
244
	fi
Camil Staps's avatar
Camil Staps committed
245

246
	if [ $PROFILE -ne 0 ]; then
247
		google-pprof --pdf $IP /tmp/prof.out > $MODULE.prof.pdf
248 249
	fi

250 251
	[ "$OS" == "Windows_NT" ] && dos2unix $MODULE.result

252
	if [ $BENCHMARK -gt 0 ] && [ -f "$MODULE.bm$EXPECTED_PREFIX.expected" ]; then
253
		git diff --no-index --word-diff -U0 $MODULE.bm$EXPECTED_PREFIX.expected $MODULE.result
254
	else
255
		git diff --no-index --word-diff -U0 $MODULE$EXPECTED_PREFIX.expected $MODULE.result
256
	fi
Camil Staps's avatar
Camil Staps committed
257
	if [ $? -ne 0 ]; then
258
		echo -e "${RED}FAILED: $MODULE (different result)$RESET"
Camil Staps's avatar
Camil Staps committed
259
		FAILED+=("$MODULE")
260 261
	else
		echo -e "${GREEN}Passed: $MODULE$RESET"
Camil Staps's avatar
Camil Staps committed
262
	fi
Camil Staps's avatar
Camil Staps committed
263
done
264

Camil Staps's avatar
Camil Staps committed
265
if [ ${#FAILED[@]} -eq 0 ]; then
Camil Staps's avatar
Camil Staps committed
266
	echo -e "${GREEN}All tests passed$RESET"
Camil Staps's avatar
Camil Staps committed
267
else
Camil Staps's avatar
Camil Staps committed
268
	echo -e "${RED}Some tests failed: ${FAILED[@]}$RESET"
Camil Staps's avatar
Camil Staps committed
269
	exit 1
Camil Staps's avatar
Camil Staps committed
270
fi