1 #**************************************************************************
2 #* *
3 #* OCaml *
4 #* *
5 #* Xavier Clerc, SED, INRIA Rocquencourt *
6 #* *
7 #* Copyright 2010 Institut National de Recherche en Informatique et *
8 #* en Automatique. *
9 #* *
10 #* All rights reserved. This file is distributed under the terms of *
11 #* the GNU Lesser General Public License version 2.1, with the *
12 #* special exception on linking described in the file LICENSE. *
13 #* *
14 #**************************************************************************
15
16 BASEDIR := $(shell pwd)
17 NO_PRINT=`$(MAKE) empty --no-print-directory >/dev/null 2>&1 \
18 && echo --no-print-directory`
19
20 FIND=find
21 TOPDIR := ..
22 include $(TOPDIR)/Makefile.tools
23
24 OCAMLTESTDIR_CYGPATH=$(shell $(CYGPATH) $(BASEDIR)/$(DIR)/_ocamltest)
25
26 failstamp := failure.stamp
27
28 TESTLOG ?= _log
29
30 ocamltest_directory := ../ocamltest
31
32 ocamltest_program := $(or \
33 $(wildcard $(ocamltest_directory)/ocamltest.opt$(EXE)),\
34 $(wildcard $(ocamltest_directory)/ocamltest$(EXE)))
35
36 ifeq "$(UNIX_OR_WIN32)" "unix"
37 ifeq "$(SYSTEM)" "cygwin"
38 find := /usr/bin/find
39 else # Non-cygwin Unix
40 find := find
41 endif
42 FLEXLINK_ENV =
43 else # Windows
44 find := /usr/bin/find
45 FLEXDLL_SUBMODULE_PRESENT := $(wildcard ../flexdll/Makefile)
46 ifeq "$(FLEXDLL_SUBMODULE_PRESENT)" ""
47 FLEXLINK_ENV =
48 else
49 ROOT := $(shell cd .. && pwd| cygpath -m -f -)
50 FLEXLINK_ENV = \
51 OCAML_FLEXLINK="$(ROOT)/boot/ocamlrun $(ROOT)/flexdll/flexlink.exe"
52 endif
53 endif
54
55 ifeq "$(FLEXLINK_ENV)" ""
56 ocamltest := MKDLL="$(MKDLL)" SORT=$(SORT) MAKE=$(MAKE) $(ocamltest_program)
57 else
58 MKDLL=$(WINTOPDIR)/boot/ocamlrun $(WINTOPDIR)/flexdll/flexlink.exe \
59 $(FLEXLINK_FLAGS)
60
61 ocamltest := $(FLEXLINK_ENV) MKDLL="$(MKDLL)" SORT=$(SORT) MAKE=$(MAKE) \
62 $(ocamltest_program)
63 endif
64
65 .PHONY: default
66 default:
67 @echo "Available targets:"
68 @echo " all launch all tests"
69 @echo " all-foo launch all tests beginning with foo"
70 @echo " parallel launch all tests using GNU parallel"
71 @echo " parallel-foo launch all tests beginning with foo using \
72 GNU parallel"
73 @echo " list FILE=f launch the tests listed in f (one per line)"
74 @echo " one DIR=p launch the tests located in path p"
75 @echo " promote DIR=p promote the reference files for the tests in p"
76 @echo " lib build library modules"
77 @echo " tools build test tools"
78 @echo " clean delete generated files"
79 @echo " report print the report for the last execution"
80 @echo
81 @echo "all*, parallel* and list can automatically re-run failed test"
82 @echo "directories if MAX_TESTSUITE_DIR_RETRIES permits"
83 @echo "(default value = $(MAX_TESTSUITE_DIR_RETRIES))"
84
85 .PHONY: all
86 all:
87 @rm -f $(TESTLOG)
88 @$(MAKE) $(NO_PRINT) new-without-report
89 @$(MAKE) $(NO_PRINT) report
90
91 .PHONY: new-without-report
92 new-without-report: lib tools
93 @rm -f $(failstamp)
94 @(IFS=$$(printf "\r\n"); \
95 $(ocamltest) -find-test-dirs tests | while read dir; do \
96 echo Running tests from \'$$dir\' ... ; \
97 $(MAKE) exec-ocamltest DIR=$$dir \
98 OCAMLTESTENV="" OCAMLTESTFLAGS=""; \
99 done || echo outer loop >> $(failstamp)) 2>&1 | tee -a $(TESTLOG)
100 @$(MAKE) check-failstamp
101
102 .PHONY: check-failstamp
103 check-failstamp:
104 @if [ -f $(failstamp) ]; then \
105 echo 'Unexpected error in the test infrastructure:'; \
106 cat $(failstamp); \
107 rm $(failstamp); \
108 exit 1; \
109 fi
110
111 .PHONY: all-%
112 all-%: lib tools
113 @for dir in tests/$**; do \
114 $(MAKE) $(NO_PRINT) exec-one DIR=$$dir; \
115 done 2>&1 | tee $(TESTLOG)
116 @$(MAKE) $(NO_PRINT) retries
117 @$(MAKE) report
118
119 # The targets below use GNU parallel to parallelize tests
120 # 'make all' and 'make parallel' should be equivalent
121 #
122 # parallel uses specific logic to make sure the output of the commands
123 # run in parallel are not mangled. By default, it will reproduce
124 # the output of each completed command atomically, in order of completion.
125 #
126 # With the --keep-order option, we ask it to save the completed output
127 # and replay them in invocation order instead. In theory this costs
128 # a tiny bit of performance, but I could not measure any difference.
129 # In theory again, the reporting logic works fine with test outputs
130 # coming in in arbitrary order (so we should not need --keep-order),
131 # but keeping the output deterministic is guaranteed to make
132 # someone's life easier at least once in the future.
133 #
134 # Finally, note that the command we run has a 2>&1 redirection, as
135 # in the other make targets. If we removed the quoting around
136 # "$(MAKE) ... 2>&1", the rediction would apply to the complete output
137 # of parallel, and have a slightly different behavior: by default parallel
138 # cleanly separates the stdout and stderr output of each completed command,
139 # printing stderr first then stdout second (for each command).
140 # I chose to keep the previous behavior exactly unchanged,
141 # but the demangling separation is arguably nicer behavior that we might
142 # want to implement at the exec-one level to also have it in the 'all' target.
143 .PHONY: parallel-%
144 parallel-%: lib tools
145 @echo | parallel >/dev/null 2>/dev/null \
146 || (echo "Unable to run the GNU parallel tool;";\
147 echo "You should install it before using the parallel* targets.";\
148 exit 1)
149 @echo | parallel --gnu --no-notice >/dev/null 2>/dev/null \
150 || (echo "Your 'parallel' tool seems incompatible with GNU parallel.";\
151 echo "This target requires GNU parallel.";\
152 exit 1)
153 @for dir in tests/$**; do echo $$dir; done \
154 | parallel --gnu --no-notice --keep-order \
155 "$(MAKE) $(NO_PRINT) exec-one DIR={} 2>&1" \
156 | tee $(TESTLOG)
157 @$(MAKE) $(NO_PRINT) retries
158 @$(MAKE) report
159
160 .PHONY: parallel
161 parallel: parallel-*
162
163 .PHONY: list
164 list: lib tools
165 @if [ -z "$(FILE)" ]; \
166 then echo "No value set for variable 'FILE'."; \
167 exit 1; \
168 fi
169 @while read LINE; do \
170 $(MAKE) $(NO_PRINT) exec-one DIR=$$LINE; \
171 done <$(FILE) 2>&1 | tee $(TESTLOG)
172 @$(MAKE) $(NO_PRINT) retries
173 @$(MAKE) report
174
175 .PHONY: one
176 one: lib tools
177 @if [ -z "$(DIR)" ]; then \
178 echo "No value set for variable 'DIR'."; \
179 exit 1; \
180 fi
181 @if [ ! -d $(DIR) ]; then \
182 echo "Directory '$(DIR)' does not exist."; \
183 exit 1; \
184 fi
185 @$(MAKE) $(NO_PRINT) exec-one DIR=$(DIR)
186 @$(MAKE) check-failstamp
187
188 .PHONY: exec-one
189 exec-one:
190 @if $(ocamltest) -list-tests $(DIR) >/dev/null 2>&1; then \
191 echo "Running tests from '$$DIR' ..."; \
192 $(MAKE) exec-ocamltest DIR=$(DIR) \
193 OCAMLTESTENV="OCAMLTESTDIR=$(OCAMLTESTDIR_CYGPATH)" \
194 OCAMLTESTFLAGS=""; \
195 else \
196 for dir in $(DIR)/*; do \
197 if [ -d $$dir ]; then \
198 $(MAKE) exec-one DIR=$$dir; \
199 fi; \
200 done; \
201 fi
202
203 .PHONY: exec-ocamltest
204 exec-ocamltest:
205 @if [ -z "$(DIR)" ]; then exit 1; fi
206 @if [ ! -d "$(DIR)" ]; then exit 1; fi
207 @(IFS=$$(printf "\r\n"); \
208 $(ocamltest) -list-tests $(DIR) | while read testfile; do \
209 TERM=dumb $(OCAMLTESTENV) \
210 $(ocamltest) $(OCAMLTESTFLAGS) $(DIR)/$$testfile || \
211 echo " ... testing '$$testfile' => unexpected error"; \
212 done) || echo directory "$(DIR)" >>$(failstamp)
213
214 .PHONY: clean-one
215 clean-one:
216 @if [ ! -f $(DIR)/Makefile ]; then \
217 for dir in $(DIR)/*; do \
218 if [ -d $$dir ]; then \
219 $(MAKE) clean-one DIR=$$dir; \
220 fi; \
221 done; \
222 else \
223 cd $(DIR) && $(MAKE) TERM=dumb BASEDIR=$(BASEDIR) clean; \
224 fi
225
226 .PHONY: promote
227 promote:
228 @if [ -z "$(DIR)" ]; then \
229 echo "No value set for variable 'DIR'."; \
230 exit 1; \
231 fi
232 @if [ ! -d $(DIR) ]; then \
233 echo "Directory '$(DIR)' does not exist."; \
234 exit 1; \
235 fi
236 @if $(ocamltest) -list-tests $(DIR) >/dev/null 2>&1; then \
237 $(MAKE) exec-ocamltest DIR=$(DIR) \
238 OCAMLTESTENV="OCAMLTESTDIR=$(OCAMLTESTDIR_CYGPATH)" \
239 OCAMLTESTFLAGS="-promote"; \
240 else \
241 cd $(DIR) && $(MAKE) TERM=dumb BASEDIR=$(BASEDIR) promote; \
242 fi
243
244 .PHONY: lib
245 lib:
246 @$(MAKE) -s -C lib
247
248 .PHONY: tools
249 tools:
250 @cd tools && $(MAKE) -s BASEDIR=$(BASEDIR)
251
252 .PHONY: clean
253 clean:
254 @$(MAKE) -C lib clean
255 @cd tools && $(MAKE) BASEDIR=$(BASEDIR) clean
256 $(FIND) . -name '*_ocamltest*' | xargs rm -rf
257 rm -f $(failstamp)
258
259 .PHONY: report
260 report:
261 @if [ ! -f $(TESTLOG) ]; then echo "No $(TESTLOG) file."; exit 1; fi
262 @awk -f summarize.awk < $(TESTLOG)
263
264 .PHONY: retry-list
265 retry-list:
266 @while read LINE; do \
267 if [ -n "$$LINE" ] ; then \
268 echo re-ran $$LINE>> $(TESTLOG); \
269 $(MAKE) $(NO_PRINT) clean-one DIR=$$LINE; \
270 $(MAKE) $(NO_PRINT) exec-one DIR=$$LINE 2>&1 | tee -a $(TESTLOG) ; \
271 fi \
272 done <_retries;
273 @$(MAKE) $(NO_PRINT) retries
274
275 .PHONY: retries
276 retries:
277 @awk -v retries=1 -v max_retries=$(MAX_TESTSUITE_DIR_RETRIES) \
278 -f summarize.awk < $(TESTLOG) > _retries
279 @test `cat _retries | wc -l` -eq 0 || $(MAKE) $(NO_PRINT) retry-list
280 @rm -f _retries
281
282 .PHONY: empty
283 empty:
284