284 lines | 9298 chars
| 1 | #************************************************************************** |
| 2 | #* * |
| 3 | #* OCaml * |
| 4 | #* * |
| 5 | #* Xavier Clerc, SED, INRIA Rocquencourt * |
| 6 | #* * |
| 7 | #* Copyright 2010 Institut National de Recherche en Informatique et * |
| 8 | #* en Automatique. * |
| 9 | #* * |
| 10 | #* All rights reserved. This file is distributed under the terms of * |
| 11 | #* the GNU Lesser General Public License version 2.1, with the * |
| 12 | #* special exception on linking described in the file LICENSE. * |
| 13 | #* * |
| 14 | #************************************************************************** |
| 15 | |
| 16 | BASEDIR := $(shell pwd) |
| 17 | NO_PRINT=`$(MAKE) empty --no-print-directory >/dev/null 2>&1 \ |
| 18 | && echo --no-print-directory` |
| 19 | |
| 20 | FIND=find |
| 21 | TOPDIR := .. |
| 22 | include $(TOPDIR)/Makefile.tools |
| 23 | |
| 24 | OCAMLTESTDIR_CYGPATH=$(shell $(CYGPATH) $(BASEDIR)/$(DIR)/_ocamltest) |
| 25 | |
| 26 | failstamp := failure.stamp |
| 27 | |
| 28 | TESTLOG ?= _log |
| 29 | |
| 30 | ocamltest_directory := ../ocamltest |
| 31 | |
| 32 | ocamltest_program := $(or \ |
| 33 | $(wildcard $(ocamltest_directory)/ocamltest.opt$(EXE)),\ |
| 34 | $(wildcard $(ocamltest_directory)/ocamltest$(EXE))) |
| 35 | |
| 36 | ifeq "$(UNIX_OR_WIN32)" "unix" |
| 37 | ifeq "$(SYSTEM)" "cygwin" |
| 38 | find := /usr/bin/find |
| 39 | else # Non-cygwin Unix |
| 40 | find := find |
| 41 | endif |
| 42 | FLEXLINK_ENV = |
| 43 | else # Windows |
| 44 | find := /usr/bin/find |
| 45 | FLEXDLL_SUBMODULE_PRESENT := $(wildcard ../flexdll/Makefile) |
| 46 | ifeq "$(FLEXDLL_SUBMODULE_PRESENT)" "" |
| 47 | FLEXLINK_ENV = |
| 48 | else |
| 49 | ROOT := $(shell cd .. && pwd| cygpath -m -f -) |
| 50 | FLEXLINK_ENV = \ |
| 51 | OCAML_FLEXLINK="$(ROOT)/boot/ocamlrun $(ROOT)/flexdll/flexlink.exe" |
| 52 | endif |
| 53 | endif |
| 54 | |
| 55 | ifeq "$(FLEXLINK_ENV)" "" |
| 56 | ocamltest := MKDLL="$(MKDLL)" SORT=$(SORT) MAKE=$(MAKE) $(ocamltest_program) |
| 57 | else |
| 58 | MKDLL=$(WINTOPDIR)/boot/ocamlrun $(WINTOPDIR)/flexdll/flexlink.exe \ |
| 59 | $(FLEXLINK_FLAGS) |
| 60 | |
| 61 | ocamltest := $(FLEXLINK_ENV) MKDLL="$(MKDLL)" SORT=$(SORT) MAKE=$(MAKE) \ |
| 62 | $(ocamltest_program) |
| 63 | endif |
| 64 | |
| 65 | .PHONY: default |
| 66 | default: |
| 67 | @echo "Available targets:" |
| 68 | @echo " all launch all tests" |
| 69 | @echo " all-foo launch all tests beginning with foo" |
| 70 | @echo " parallel launch all tests using GNU parallel" |
| 71 | @echo " parallel-foo launch all tests beginning with foo using \ |
| 72 | GNU parallel" |
| 73 | @echo " list FILE=f launch the tests listed in f (one per line)" |
| 74 | @echo " one DIR=p launch the tests located in path p" |
| 75 | @echo " promote DIR=p promote the reference files for the tests in p" |
| 76 | @echo " lib build library modules" |
| 77 | @echo " tools build test tools" |
| 78 | @echo " clean delete generated files" |
| 79 | @echo " report print the report for the last execution" |
| 80 | @echo |
| 81 | @echo "all*, parallel* and list can automatically re-run failed test" |
| 82 | @echo "directories if MAX_TESTSUITE_DIR_RETRIES permits" |
| 83 | @echo "(default value = $(MAX_TESTSUITE_DIR_RETRIES))" |
| 84 | |
| 85 | .PHONY: all |
| 86 | all: |
| 87 | @rm -f $(TESTLOG) |
| 88 | @$(MAKE) $(NO_PRINT) new-without-report |
| 89 | @$(MAKE) $(NO_PRINT) report |
| 90 | |
| 91 | .PHONY: new-without-report |
| 92 | new-without-report: lib tools |
| 93 | @rm -f $(failstamp) |
| 94 | @(IFS=$$(printf "\r\n"); \ |
| 95 | $(ocamltest) -find-test-dirs tests | while read dir; do \ |
| 96 | echo Running tests from \'$$dir\' ... ; \ |
| 97 | $(MAKE) exec-ocamltest DIR=$$dir \ |
| 98 | OCAMLTESTENV="" OCAMLTESTFLAGS=""; \ |
| 99 | done || echo outer loop >> $(failstamp)) 2>&1 | tee -a $(TESTLOG) |
| 100 | @$(MAKE) check-failstamp |
| 101 | |
| 102 | .PHONY: check-failstamp |
| 103 | check-failstamp: |
| 104 | @if [ -f $(failstamp) ]; then \ |
| 105 | echo 'Unexpected error in the test infrastructure:'; \ |
| 106 | cat $(failstamp); \ |
| 107 | rm $(failstamp); \ |
| 108 | exit 1; \ |
| 109 | fi |
| 110 | |
| 111 | .PHONY: all-% |
| 112 | all-%: lib tools |
| 113 | @for dir in tests/$**; do \ |
| 114 | $(MAKE) $(NO_PRINT) exec-one DIR=$$dir; \ |
| 115 | done 2>&1 | tee $(TESTLOG) |
| 116 | @$(MAKE) $(NO_PRINT) retries |
| 117 | @$(MAKE) report |
| 118 | |
| 119 | # The targets below use GNU parallel to parallelize tests |
| 120 | # 'make all' and 'make parallel' should be equivalent |
| 121 | # |
| 122 | # parallel uses specific logic to make sure the output of the commands |
| 123 | # run in parallel are not mangled. By default, it will reproduce |
| 124 | # the output of each completed command atomically, in order of completion. |
| 125 | # |
| 126 | # With the --keep-order option, we ask it to save the completed output |
| 127 | # and replay them in invocation order instead. In theory this costs |
| 128 | # a tiny bit of performance, but I could not measure any difference. |
| 129 | # In theory again, the reporting logic works fine with test outputs |
| 130 | # coming in in arbitrary order (so we should not need --keep-order), |
| 131 | # but keeping the output deterministic is guaranteed to make |
| 132 | # someone's life easier at least once in the future. |
| 133 | # |
| 134 | # Finally, note that the command we run has a 2>&1 redirection, as |
| 135 | # in the other make targets. If we removed the quoting around |
| 136 | # "$(MAKE) ... 2>&1", the rediction would apply to the complete output |
| 137 | # of parallel, and have a slightly different behavior: by default parallel |
| 138 | # cleanly separates the stdout and stderr output of each completed command, |
| 139 | # printing stderr first then stdout second (for each command). |
| 140 | # I chose to keep the previous behavior exactly unchanged, |
| 141 | # but the demangling separation is arguably nicer behavior that we might |
| 142 | # want to implement at the exec-one level to also have it in the 'all' target. |
| 143 | .PHONY: parallel-% |
| 144 | parallel-%: lib tools |
| 145 | @echo | parallel >/dev/null 2>/dev/null \ |
| 146 | || (echo "Unable to run the GNU parallel tool;";\ |
| 147 | echo "You should install it before using the parallel* targets.";\ |
| 148 | exit 1) |
| 149 | @echo | parallel --gnu --no-notice >/dev/null 2>/dev/null \ |
| 150 | || (echo "Your 'parallel' tool seems incompatible with GNU parallel.";\ |
| 151 | echo "This target requires GNU parallel.";\ |
| 152 | exit 1) |
| 153 | @for dir in tests/$**; do echo $$dir; done \ |
| 154 | | parallel --gnu --no-notice --keep-order \ |
| 155 | "$(MAKE) $(NO_PRINT) exec-one DIR={} 2>&1" \ |
| 156 | | tee $(TESTLOG) |
| 157 | @$(MAKE) $(NO_PRINT) retries |
| 158 | @$(MAKE) report |
| 159 | |
| 160 | .PHONY: parallel |
| 161 | parallel: parallel-* |
| 162 | |
| 163 | .PHONY: list |
| 164 | list: lib tools |
| 165 | @if [ -z "$(FILE)" ]; \ |
| 166 | then echo "No value set for variable 'FILE'."; \ |
| 167 | exit 1; \ |
| 168 | fi |
| 169 | @while read LINE; do \ |
| 170 | $(MAKE) $(NO_PRINT) exec-one DIR=$$LINE; \ |
| 171 | done <$(FILE) 2>&1 | tee $(TESTLOG) |
| 172 | @$(MAKE) $(NO_PRINT) retries |
| 173 | @$(MAKE) report |
| 174 | |
| 175 | .PHONY: one |
| 176 | one: lib tools |
| 177 | @if [ -z "$(DIR)" ]; then \ |
| 178 | echo "No value set for variable 'DIR'."; \ |
| 179 | exit 1; \ |
| 180 | fi |
| 181 | @if [ ! -d $(DIR) ]; then \ |
| 182 | echo "Directory '$(DIR)' does not exist."; \ |
| 183 | exit 1; \ |
| 184 | fi |
| 185 | @$(MAKE) $(NO_PRINT) exec-one DIR=$(DIR) |
| 186 | @$(MAKE) check-failstamp |
| 187 | |
| 188 | .PHONY: exec-one |
| 189 | exec-one: |
| 190 | @if $(ocamltest) -list-tests $(DIR) >/dev/null 2>&1; then \ |
| 191 | echo "Running tests from '$$DIR' ..."; \ |
| 192 | $(MAKE) exec-ocamltest DIR=$(DIR) \ |
| 193 | OCAMLTESTENV="OCAMLTESTDIR=$(OCAMLTESTDIR_CYGPATH)" \ |
| 194 | OCAMLTESTFLAGS=""; \ |
| 195 | else \ |
| 196 | for dir in $(DIR)/*; do \ |
| 197 | if [ -d $$dir ]; then \ |
| 198 | $(MAKE) exec-one DIR=$$dir; \ |
| 199 | fi; \ |
| 200 | done; \ |
| 201 | fi |
| 202 | |
| 203 | .PHONY: exec-ocamltest |
| 204 | exec-ocamltest: |
| 205 | @if [ -z "$(DIR)" ]; then exit 1; fi |
| 206 | @if [ ! -d "$(DIR)" ]; then exit 1; fi |
| 207 | @(IFS=$$(printf "\r\n"); \ |
| 208 | $(ocamltest) -list-tests $(DIR) | while read testfile; do \ |
| 209 | TERM=dumb $(OCAMLTESTENV) \ |
| 210 | $(ocamltest) $(OCAMLTESTFLAGS) $(DIR)/$$testfile || \ |
| 211 | echo " ... testing '$$testfile' => unexpected error"; \ |
| 212 | done) || echo directory "$(DIR)" >>$(failstamp) |
| 213 | |
| 214 | .PHONY: clean-one |
| 215 | clean-one: |
| 216 | @if [ ! -f $(DIR)/Makefile ]; then \ |
| 217 | for dir in $(DIR)/*; do \ |
| 218 | if [ -d $$dir ]; then \ |
| 219 | $(MAKE) clean-one DIR=$$dir; \ |
| 220 | fi; \ |
| 221 | done; \ |
| 222 | else \ |
| 223 | cd $(DIR) && $(MAKE) TERM=dumb BASEDIR=$(BASEDIR) clean; \ |
| 224 | fi |
| 225 | |
| 226 | .PHONY: promote |
| 227 | promote: |
| 228 | @if [ -z "$(DIR)" ]; then \ |
| 229 | echo "No value set for variable 'DIR'."; \ |
| 230 | exit 1; \ |
| 231 | fi |
| 232 | @if [ ! -d $(DIR) ]; then \ |
| 233 | echo "Directory '$(DIR)' does not exist."; \ |
| 234 | exit 1; \ |
| 235 | fi |
| 236 | @if $(ocamltest) -list-tests $(DIR) >/dev/null 2>&1; then \ |
| 237 | $(MAKE) exec-ocamltest DIR=$(DIR) \ |
| 238 | OCAMLTESTENV="OCAMLTESTDIR=$(OCAMLTESTDIR_CYGPATH)" \ |
| 239 | OCAMLTESTFLAGS="-promote"; \ |
| 240 | else \ |
| 241 | cd $(DIR) && $(MAKE) TERM=dumb BASEDIR=$(BASEDIR) promote; \ |
| 242 | fi |
| 243 | |
| 244 | .PHONY: lib |
| 245 | lib: |
| 246 | @$(MAKE) -s -C lib |
| 247 | |
| 248 | .PHONY: tools |
| 249 | tools: |
| 250 | @cd tools && $(MAKE) -s BASEDIR=$(BASEDIR) |
| 251 | |
| 252 | .PHONY: clean |
| 253 | clean: |
| 254 | @$(MAKE) -C lib clean |
| 255 | @cd tools && $(MAKE) BASEDIR=$(BASEDIR) clean |
| 256 | $(FIND) . -name '*_ocamltest*' | xargs rm -rf |
| 257 | rm -f $(failstamp) |
| 258 | |
| 259 | .PHONY: report |
| 260 | report: |
| 261 | @if [ ! -f $(TESTLOG) ]; then echo "No $(TESTLOG) file."; exit 1; fi |
| 262 | @awk -f summarize.awk < $(TESTLOG) |
| 263 | |
| 264 | .PHONY: retry-list |
| 265 | retry-list: |
| 266 | @while read LINE; do \ |
| 267 | if [ -n "$$LINE" ] ; then \ |
| 268 | echo re-ran $$LINE>> $(TESTLOG); \ |
| 269 | $(MAKE) $(NO_PRINT) clean-one DIR=$$LINE; \ |
| 270 | $(MAKE) $(NO_PRINT) exec-one DIR=$$LINE 2>&1 | tee -a $(TESTLOG) ; \ |
| 271 | fi \ |
| 272 | done <_retries; |
| 273 | @$(MAKE) $(NO_PRINT) retries |
| 274 | |
| 275 | .PHONY: retries |
| 276 | retries: |
| 277 | @awk -v retries=1 -v max_retries=$(MAX_TESTSUITE_DIR_RETRIES) \ |
| 278 | -f summarize.awk < $(TESTLOG) > _retries |
| 279 | @test `cat _retries | wc -l` -eq 0 || $(MAKE) $(NO_PRINT) retry-list |
| 280 | @rm -f _retries |
| 281 | |
| 282 | .PHONY: empty |
| 283 | empty: |
| 284 |