284 lines | 9298 chars
1 | #************************************************************************** |
2 | #* * |
3 | #* OCaml * |
4 | #* * |
5 | #* Xavier Clerc, SED, INRIA Rocquencourt * |
6 | #* * |
7 | #* Copyright 2010 Institut National de Recherche en Informatique et * |
8 | #* en Automatique. * |
9 | #* * |
10 | #* All rights reserved. This file is distributed under the terms of * |
11 | #* the GNU Lesser General Public License version 2.1, with the * |
12 | #* special exception on linking described in the file LICENSE. * |
13 | #* * |
14 | #************************************************************************** |
15 | |
16 | BASEDIR := $(shell pwd) |
17 | NO_PRINT=`$(MAKE) empty --no-print-directory >/dev/null 2>&1 \ |
18 | && echo --no-print-directory` |
19 | |
20 | FIND=find |
21 | TOPDIR := .. |
22 | include $(TOPDIR)/Makefile.tools |
23 | |
24 | OCAMLTESTDIR_CYGPATH=$(shell $(CYGPATH) $(BASEDIR)/$(DIR)/_ocamltest) |
25 | |
26 | failstamp := failure.stamp |
27 | |
28 | TESTLOG ?= _log |
29 | |
30 | ocamltest_directory := ../ocamltest |
31 | |
32 | ocamltest_program := $(or \ |
33 | $(wildcard $(ocamltest_directory)/ocamltest.opt$(EXE)),\ |
34 | $(wildcard $(ocamltest_directory)/ocamltest$(EXE))) |
35 | |
36 | ifeq "$(UNIX_OR_WIN32)" "unix" |
37 | ifeq "$(SYSTEM)" "cygwin" |
38 | find := /usr/bin/find |
39 | else # Non-cygwin Unix |
40 | find := find |
41 | endif |
42 | FLEXLINK_ENV = |
43 | else # Windows |
44 | find := /usr/bin/find |
45 | FLEXDLL_SUBMODULE_PRESENT := $(wildcard ../flexdll/Makefile) |
46 | ifeq "$(FLEXDLL_SUBMODULE_PRESENT)" "" |
47 | FLEXLINK_ENV = |
48 | else |
49 | ROOT := $(shell cd .. && pwd| cygpath -m -f -) |
50 | FLEXLINK_ENV = \ |
51 | OCAML_FLEXLINK="$(ROOT)/boot/ocamlrun $(ROOT)/flexdll/flexlink.exe" |
52 | endif |
53 | endif |
54 | |
55 | ifeq "$(FLEXLINK_ENV)" "" |
56 | ocamltest := MKDLL="$(MKDLL)" SORT=$(SORT) MAKE=$(MAKE) $(ocamltest_program) |
57 | else |
58 | MKDLL=$(WINTOPDIR)/boot/ocamlrun $(WINTOPDIR)/flexdll/flexlink.exe \ |
59 | $(FLEXLINK_FLAGS) |
60 | |
61 | ocamltest := $(FLEXLINK_ENV) MKDLL="$(MKDLL)" SORT=$(SORT) MAKE=$(MAKE) \ |
62 | $(ocamltest_program) |
63 | endif |
64 | |
65 | .PHONY: default |
66 | default: |
67 | @echo "Available targets:" |
68 | @echo " all launch all tests" |
69 | @echo " all-foo launch all tests beginning with foo" |
70 | @echo " parallel launch all tests using GNU parallel" |
71 | @echo " parallel-foo launch all tests beginning with foo using \ |
72 | GNU parallel" |
73 | @echo " list FILE=f launch the tests listed in f (one per line)" |
74 | @echo " one DIR=p launch the tests located in path p" |
75 | @echo " promote DIR=p promote the reference files for the tests in p" |
76 | @echo " lib build library modules" |
77 | @echo " tools build test tools" |
78 | @echo " clean delete generated files" |
79 | @echo " report print the report for the last execution" |
80 | @echo |
81 | @echo "all*, parallel* and list can automatically re-run failed test" |
82 | @echo "directories if MAX_TESTSUITE_DIR_RETRIES permits" |
83 | @echo "(default value = $(MAX_TESTSUITE_DIR_RETRIES))" |
84 | |
85 | .PHONY: all |
86 | all: |
87 | @rm -f $(TESTLOG) |
88 | @$(MAKE) $(NO_PRINT) new-without-report |
89 | @$(MAKE) $(NO_PRINT) report |
90 | |
91 | .PHONY: new-without-report |
92 | new-without-report: lib tools |
93 | @rm -f $(failstamp) |
94 | @(IFS=$$(printf "\r\n"); \ |
95 | $(ocamltest) -find-test-dirs tests | while read dir; do \ |
96 | echo Running tests from \'$$dir\' ... ; \ |
97 | $(MAKE) exec-ocamltest DIR=$$dir \ |
98 | OCAMLTESTENV="" OCAMLTESTFLAGS=""; \ |
99 | done || echo outer loop >> $(failstamp)) 2>&1 | tee -a $(TESTLOG) |
100 | @$(MAKE) check-failstamp |
101 | |
102 | .PHONY: check-failstamp |
103 | check-failstamp: |
104 | @if [ -f $(failstamp) ]; then \ |
105 | echo 'Unexpected error in the test infrastructure:'; \ |
106 | cat $(failstamp); \ |
107 | rm $(failstamp); \ |
108 | exit 1; \ |
109 | fi |
110 | |
111 | .PHONY: all-% |
112 | all-%: lib tools |
113 | @for dir in tests/$**; do \ |
114 | $(MAKE) $(NO_PRINT) exec-one DIR=$$dir; \ |
115 | done 2>&1 | tee $(TESTLOG) |
116 | @$(MAKE) $(NO_PRINT) retries |
117 | @$(MAKE) report |
118 | |
119 | # The targets below use GNU parallel to parallelize tests |
120 | # 'make all' and 'make parallel' should be equivalent |
121 | # |
122 | # parallel uses specific logic to make sure the output of the commands |
123 | # run in parallel are not mangled. By default, it will reproduce |
124 | # the output of each completed command atomically, in order of completion. |
125 | # |
126 | # With the --keep-order option, we ask it to save the completed output |
127 | # and replay them in invocation order instead. In theory this costs |
128 | # a tiny bit of performance, but I could not measure any difference. |
129 | # In theory again, the reporting logic works fine with test outputs |
130 | # coming in in arbitrary order (so we should not need --keep-order), |
131 | # but keeping the output deterministic is guaranteed to make |
132 | # someone's life easier at least once in the future. |
133 | # |
134 | # Finally, note that the command we run has a 2>&1 redirection, as |
135 | # in the other make targets. If we removed the quoting around |
136 | # "$(MAKE) ... 2>&1", the rediction would apply to the complete output |
137 | # of parallel, and have a slightly different behavior: by default parallel |
138 | # cleanly separates the stdout and stderr output of each completed command, |
139 | # printing stderr first then stdout second (for each command). |
140 | # I chose to keep the previous behavior exactly unchanged, |
141 | # but the demangling separation is arguably nicer behavior that we might |
142 | # want to implement at the exec-one level to also have it in the 'all' target. |
143 | .PHONY: parallel-% |
144 | parallel-%: lib tools |
145 | @echo | parallel >/dev/null 2>/dev/null \ |
146 | || (echo "Unable to run the GNU parallel tool;";\ |
147 | echo "You should install it before using the parallel* targets.";\ |
148 | exit 1) |
149 | @echo | parallel --gnu --no-notice >/dev/null 2>/dev/null \ |
150 | || (echo "Your 'parallel' tool seems incompatible with GNU parallel.";\ |
151 | echo "This target requires GNU parallel.";\ |
152 | exit 1) |
153 | @for dir in tests/$**; do echo $$dir; done \ |
154 | | parallel --gnu --no-notice --keep-order \ |
155 | "$(MAKE) $(NO_PRINT) exec-one DIR={} 2>&1" \ |
156 | | tee $(TESTLOG) |
157 | @$(MAKE) $(NO_PRINT) retries |
158 | @$(MAKE) report |
159 | |
160 | .PHONY: parallel |
161 | parallel: parallel-* |
162 | |
163 | .PHONY: list |
164 | list: lib tools |
165 | @if [ -z "$(FILE)" ]; \ |
166 | then echo "No value set for variable 'FILE'."; \ |
167 | exit 1; \ |
168 | fi |
169 | @while read LINE; do \ |
170 | $(MAKE) $(NO_PRINT) exec-one DIR=$$LINE; \ |
171 | done <$(FILE) 2>&1 | tee $(TESTLOG) |
172 | @$(MAKE) $(NO_PRINT) retries |
173 | @$(MAKE) report |
174 | |
175 | .PHONY: one |
176 | one: lib tools |
177 | @if [ -z "$(DIR)" ]; then \ |
178 | echo "No value set for variable 'DIR'."; \ |
179 | exit 1; \ |
180 | fi |
181 | @if [ ! -d $(DIR) ]; then \ |
182 | echo "Directory '$(DIR)' does not exist."; \ |
183 | exit 1; \ |
184 | fi |
185 | @$(MAKE) $(NO_PRINT) exec-one DIR=$(DIR) |
186 | @$(MAKE) check-failstamp |
187 | |
188 | .PHONY: exec-one |
189 | exec-one: |
190 | @if $(ocamltest) -list-tests $(DIR) >/dev/null 2>&1; then \ |
191 | echo "Running tests from '$$DIR' ..."; \ |
192 | $(MAKE) exec-ocamltest DIR=$(DIR) \ |
193 | OCAMLTESTENV="OCAMLTESTDIR=$(OCAMLTESTDIR_CYGPATH)" \ |
194 | OCAMLTESTFLAGS=""; \ |
195 | else \ |
196 | for dir in $(DIR)/*; do \ |
197 | if [ -d $$dir ]; then \ |
198 | $(MAKE) exec-one DIR=$$dir; \ |
199 | fi; \ |
200 | done; \ |
201 | fi |
202 | |
203 | .PHONY: exec-ocamltest |
204 | exec-ocamltest: |
205 | @if [ -z "$(DIR)" ]; then exit 1; fi |
206 | @if [ ! -d "$(DIR)" ]; then exit 1; fi |
207 | @(IFS=$$(printf "\r\n"); \ |
208 | $(ocamltest) -list-tests $(DIR) | while read testfile; do \ |
209 | TERM=dumb $(OCAMLTESTENV) \ |
210 | $(ocamltest) $(OCAMLTESTFLAGS) $(DIR)/$$testfile || \ |
211 | echo " ... testing '$$testfile' => unexpected error"; \ |
212 | done) || echo directory "$(DIR)" >>$(failstamp) |
213 | |
214 | .PHONY: clean-one |
215 | clean-one: |
216 | @if [ ! -f $(DIR)/Makefile ]; then \ |
217 | for dir in $(DIR)/*; do \ |
218 | if [ -d $$dir ]; then \ |
219 | $(MAKE) clean-one DIR=$$dir; \ |
220 | fi; \ |
221 | done; \ |
222 | else \ |
223 | cd $(DIR) && $(MAKE) TERM=dumb BASEDIR=$(BASEDIR) clean; \ |
224 | fi |
225 | |
226 | .PHONY: promote |
227 | promote: |
228 | @if [ -z "$(DIR)" ]; then \ |
229 | echo "No value set for variable 'DIR'."; \ |
230 | exit 1; \ |
231 | fi |
232 | @if [ ! -d $(DIR) ]; then \ |
233 | echo "Directory '$(DIR)' does not exist."; \ |
234 | exit 1; \ |
235 | fi |
236 | @if $(ocamltest) -list-tests $(DIR) >/dev/null 2>&1; then \ |
237 | $(MAKE) exec-ocamltest DIR=$(DIR) \ |
238 | OCAMLTESTENV="OCAMLTESTDIR=$(OCAMLTESTDIR_CYGPATH)" \ |
239 | OCAMLTESTFLAGS="-promote"; \ |
240 | else \ |
241 | cd $(DIR) && $(MAKE) TERM=dumb BASEDIR=$(BASEDIR) promote; \ |
242 | fi |
243 | |
244 | .PHONY: lib |
245 | lib: |
246 | @$(MAKE) -s -C lib |
247 | |
248 | .PHONY: tools |
249 | tools: |
250 | @cd tools && $(MAKE) -s BASEDIR=$(BASEDIR) |
251 | |
252 | .PHONY: clean |
253 | clean: |
254 | @$(MAKE) -C lib clean |
255 | @cd tools && $(MAKE) BASEDIR=$(BASEDIR) clean |
256 | $(FIND) . -name '*_ocamltest*' | xargs rm -rf |
257 | rm -f $(failstamp) |
258 | |
259 | .PHONY: report |
260 | report: |
261 | @if [ ! -f $(TESTLOG) ]; then echo "No $(TESTLOG) file."; exit 1; fi |
262 | @awk -f summarize.awk < $(TESTLOG) |
263 | |
264 | .PHONY: retry-list |
265 | retry-list: |
266 | @while read LINE; do \ |
267 | if [ -n "$$LINE" ] ; then \ |
268 | echo re-ran $$LINE>> $(TESTLOG); \ |
269 | $(MAKE) $(NO_PRINT) clean-one DIR=$$LINE; \ |
270 | $(MAKE) $(NO_PRINT) exec-one DIR=$$LINE 2>&1 | tee -a $(TESTLOG) ; \ |
271 | fi \ |
272 | done <_retries; |
273 | @$(MAKE) $(NO_PRINT) retries |
274 | |
275 | .PHONY: retries |
276 | retries: |
277 | @awk -v retries=1 -v max_retries=$(MAX_TESTSUITE_DIR_RETRIES) \ |
278 | -f summarize.awk < $(TESTLOG) > _retries |
279 | @test `cat _retries | wc -l` -eq 0 || $(MAKE) $(NO_PRINT) retry-list |
280 | @rm -f _retries |
281 | |
282 | .PHONY: empty |
283 | empty: |
284 |