228 lines | 6895 chars
1 | #************************************************************************** |
2 | #* * |
3 | #* OCaml * |
4 | #* * |
5 | #* Damien Doligez, projet Gallium, INRIA Rocquencourt * |
6 | #* * |
7 | #* Copyright 2013 Institut National de Recherche en Informatique et * |
8 | #* en Automatique. * |
9 | #* * |
10 | #* All rights reserved. This file is distributed under the terms of * |
11 | #* the GNU Lesser General Public License version 2.1, with the * |
12 | #* special exception on linking described in the file LICENSE. * |
13 | #* * |
14 | #************************************************************************** |
15 | |
16 | function check() { |
17 | if (!in_test){ |
18 | printf("error at line %d: found test result without test start\n", NR); |
19 | errored = 1; |
20 | } |
21 | } |
22 | |
23 | function clear() { |
24 | curfile = ""; |
25 | in_test = 0; |
26 | } |
27 | |
28 | function record_pass() { |
29 | check(); |
30 | if (!(key in RESULTS)) ++nresults; |
31 | RESULTS[key] = "p"; |
32 | delete SKIPPED[curdir]; |
33 | clear(); |
34 | } |
35 | |
36 | function record_skip() { |
37 | check(); |
38 | if (!(key in RESULTS)) ++nresults; |
39 | RESULTS[key] = "s"; |
40 | if (curdir in SKIPPED) SKIPPED[curdir] = 1; |
41 | clear(); |
42 | } |
43 | |
44 | function record_na() { |
45 | check(); |
46 | if (!(key in RESULTS)) ++nresults; |
47 | RESULTS[key] = "n"; |
48 | if (curdir in SKIPPED) SKIPPED[curdir] = 1; |
49 | clear(); |
50 | } |
51 | |
52 | # The output cares only if the test passes at least once so if a test passes, |
53 | # but then fails in a re-run triggered by a different test, ignore it. |
54 | function record_fail() { |
55 | check(); |
56 | if (!(key in RESULTS) || RESULTS[key] == "s"){ |
57 | if (!(key in RESULTS)) ++nresults; |
58 | RESULTS[key] = "f"; |
59 | } |
60 | delete SKIPPED[curdir]; |
61 | clear(); |
62 | } |
63 | |
64 | function record_unexp() { |
65 | if (!(key in RESULTS) || RESULTS[key] == "s"){ |
66 | if (!(key in RESULTS)) ++nresults; |
67 | RESULTS[key] = "e"; |
68 | } |
69 | delete SKIPPED[curdir]; |
70 | clear(); |
71 | } |
72 | |
73 | /Running tests from '[^']*'/ { |
74 | if (in_test) record_unexp(); |
75 | match($0, /Running tests from '[^']*'/); |
76 | curdir = substr($0, RSTART+20, RLENGTH - 21); |
77 | # Use SKIPPED[curdir] as a sentinel to detect no output |
78 | SKIPPED[curdir] = 0; |
79 | key = curdir; |
80 | DIRS[key] = key; |
81 | curfile = ""; |
82 | } |
83 | |
84 | / ... testing.* ... testing/ { |
85 | printf("error at line %d: found two test results on the same line\n", NR); |
86 | errored = 1; |
87 | } |
88 | |
89 | /^ ... testing '[^']*'/ { |
90 | if (in_test) record_unexp(); |
91 | match($0, /... testing '[^']*'/); |
92 | curfile = substr($0, RSTART+13, RLENGTH-14); |
93 | if (match($0, /... testing '[^']*' with [^:=]*/)){ |
94 | curfile = substr($0, RSTART+12, RLENGTH-12); |
95 | } |
96 | key = sprintf ("%s/%s", curdir, curfile); |
97 | DIRS[key] = curdir; |
98 | in_test = 1; |
99 | } |
100 | |
101 | /^ ... testing (with|[^'])/ { |
102 | if (in_test) record_unexp(); |
103 | key = curdir; |
104 | DIRS[key] = curdir; |
105 | in_test = 1; |
106 | } |
107 | |
108 | /=> passed/ { |
109 | record_pass(); |
110 | } |
111 | |
112 | /=> skipped/ { |
113 | record_skip(); |
114 | } |
115 | |
116 | /=> n\/a/ { |
117 | record_na(); |
118 | } |
119 | |
120 | /=> failed/ { |
121 | record_fail(); |
122 | } |
123 | |
124 | /=> unexpected error/ { |
125 | record_unexp(); |
126 | } |
127 | |
128 | /^re-ran / { |
129 | if (in_test){ |
130 | printf("error at line %d: found re-ran inside a test\n", NR); |
131 | errored = 1; |
132 | }else{ |
133 | RERAN[substr($0, 8, length($0)-7)] += 1; |
134 | ++ reran; |
135 | } |
136 | } |
137 | |
138 | END { |
139 | if (errored){ |
140 | printf ("\n#### Some fatal error occurred during testing.\n\n"); |
141 | exit (3); |
142 | }else{ |
143 | if (!retries){ |
144 | for (key in SKIPPED){ |
145 | if (!SKIPPED[key]){ |
146 | ++ empty; |
147 | blanks[emptyidx++] = key; |
148 | delete SKIPPED[key]; |
149 | } |
150 | } |
151 | for (key in RESULTS){ |
152 | r = RESULTS[key]; |
153 | if (r == "p"){ |
154 | ++ passed; |
155 | }else if (r == "f"){ |
156 | ++ failed; |
157 | fail[failidx++] = key; |
158 | }else if (r == "e"){ |
159 | ++ unexped; |
160 | unexp[unexpidx++] = key; |
161 | }else if (r == "s"){ |
162 | ++ skipped; |
163 | curdir = DIRS[key]; |
164 | if (curdir in SKIPPED){ |
165 | if (SKIPPED[curdir]){ |
166 | SKIPPED[curdir] = 0; |
167 | skips[skipidx++] = curdir; |
168 | } |
169 | }else{ |
170 | skips[skipidx++] = key; |
171 | } |
172 | }else if (r == "n"){ |
173 | ++ ignored; |
174 | } |
175 | } |
176 | printf("\n"); |
177 | if (skipped != 0){ |
178 | printf("\nList of skipped tests:\n"); |
179 | for (i=0; i < skipidx; i++) printf(" %s\n", skips[i]); |
180 | } |
181 | if (empty != 0){ |
182 | printf("\nList of directories returning no results:\n"); |
183 | for (i=0; i < empty; i++) printf(" %s\n", blanks[i]); |
184 | } |
185 | if (failed != 0){ |
186 | printf("\nList of failed tests:\n"); |
187 | for (i=0; i < failed; i++) printf(" %s\n", fail[i]); |
188 | } |
189 | if (unexped != 0){ |
190 | printf("\nList of unexpected errors:\n"); |
191 | for (i=0; i < unexped; i++) printf(" %s\n", unexp[i]); |
192 | } |
193 | printf("\n"); |
194 | printf("Summary:\n"); |
195 | printf(" %3d tests passed\n", passed); |
196 | printf(" %3d tests skipped\n", skipped); |
197 | printf(" %3d tests failed\n", failed); |
198 | printf(" %3d tests not started (parent test skipped or failed)\n", |
199 | ignored); |
200 | printf(" %3d unexpected errors\n", unexped); |
201 | printf(" %3d tests considered", nresults); |
202 | if (nresults != passed + skipped + ignored + failed + unexped){ |
203 | printf (" (totals don't add up??)"); |
204 | } |
205 | printf ("\n"); |
206 | if (reran != 0){ |
207 | printf(" %3d test dir re-runs\n", reran); |
208 | } |
209 | if (failed || unexped){ |
210 | printf("#### Something failed. Exiting with error status.\n\n"); |
211 | exit 4; |
212 | } |
213 | }else{ |
214 | for (key in RESULTS){ |
215 | if (RESULTS[key] == "f" || RESULTS[key] == "e"){ |
216 | key = DIRS[key]; |
217 | if (!(key in RERUNS)){ |
218 | RERUNS[key] = 1; |
219 | if (RERAN[key] < max_retries){ |
220 | printf("%s\n", key); |
221 | } |
222 | } |
223 | } |
224 | } |
225 | } |
226 | } |
227 | } |
228 |