package / ocaml-base-compiler.4.10.0 / testsuite / summarize.awk
228 lines | 6895 chars
1 #**************************************************************************
2 #* *
3 #* OCaml *
4 #* *
5 #* Damien Doligez, projet Gallium, INRIA Rocquencourt *
6 #* *
7 #* Copyright 2013 Institut National de Recherche en Informatique et *
8 #* en Automatique. *
9 #* *
10 #* All rights reserved. This file is distributed under the terms of *
11 #* the GNU Lesser General Public License version 2.1, with the *
12 #* special exception on linking described in the file LICENSE. *
13 #* *
14 #**************************************************************************
15
16 function check() {
17 if (!in_test){
18 printf("error at line %d: found test result without test start\n", NR);
19 errored = 1;
20 }
21 }
22
23 function clear() {
24 curfile = "";
25 in_test = 0;
26 }
27
28 function record_pass() {
29 check();
30 if (!(key in RESULTS)) ++nresults;
31 RESULTS[key] = "p";
32 delete SKIPPED[curdir];
33 clear();
34 }
35
36 function record_skip() {
37 check();
38 if (!(key in RESULTS)) ++nresults;
39 RESULTS[key] = "s";
40 if (curdir in SKIPPED) SKIPPED[curdir] = 1;
41 clear();
42 }
43
44 function record_na() {
45 check();
46 if (!(key in RESULTS)) ++nresults;
47 RESULTS[key] = "n";
48 if (curdir in SKIPPED) SKIPPED[curdir] = 1;
49 clear();
50 }
51
52 # The output cares only if the test passes at least once so if a test passes,
53 # but then fails in a re-run triggered by a different test, ignore it.
54 function record_fail() {
55 check();
56 if (!(key in RESULTS) || RESULTS[key] == "s"){
57 if (!(key in RESULTS)) ++nresults;
58 RESULTS[key] = "f";
59 }
60 delete SKIPPED[curdir];
61 clear();
62 }
63
64 function record_unexp() {
65 if (!(key in RESULTS) || RESULTS[key] == "s"){
66 if (!(key in RESULTS)) ++nresults;
67 RESULTS[key] = "e";
68 }
69 delete SKIPPED[curdir];
70 clear();
71 }
72
73 /Running tests from '[^']*'/ {
74 if (in_test) record_unexp();
75 match($0, /Running tests from '[^']*'/);
76 curdir = substr($0, RSTART+20, RLENGTH - 21);
77 # Use SKIPPED[curdir] as a sentinel to detect no output
78 SKIPPED[curdir] = 0;
79 key = curdir;
80 DIRS[key] = key;
81 curfile = "";
82 }
83
84 / ... testing.* ... testing/ {
85 printf("error at line %d: found two test results on the same line\n", NR);
86 errored = 1;
87 }
88
89 /^ ... testing '[^']*'/ {
90 if (in_test) record_unexp();
91 match($0, /... testing '[^']*'/);
92 curfile = substr($0, RSTART+13, RLENGTH-14);
93 if (match($0, /... testing '[^']*' with [^:=]*/)){
94 curfile = substr($0, RSTART+12, RLENGTH-12);
95 }
96 key = sprintf ("%s/%s", curdir, curfile);
97 DIRS[key] = curdir;
98 in_test = 1;
99 }
100
101 /^ ... testing (with|[^'])/ {
102 if (in_test) record_unexp();
103 key = curdir;
104 DIRS[key] = curdir;
105 in_test = 1;
106 }
107
108 /=> passed/ {
109 record_pass();
110 }
111
112 /=> skipped/ {
113 record_skip();
114 }
115
116 /=> n\/a/ {
117 record_na();
118 }
119
120 /=> failed/ {
121 record_fail();
122 }
123
124 /=> unexpected error/ {
125 record_unexp();
126 }
127
128 /^re-ran / {
129 if (in_test){
130 printf("error at line %d: found re-ran inside a test\n", NR);
131 errored = 1;
132 }else{
133 RERAN[substr($0, 8, length($0)-7)] += 1;
134 ++ reran;
135 }
136 }
137
138 END {
139 if (errored){
140 printf ("\n#### Some fatal error occurred during testing.\n\n");
141 exit (3);
142 }else{
143 if (!retries){
144 for (key in SKIPPED){
145 if (!SKIPPED[key]){
146 ++ empty;
147 blanks[emptyidx++] = key;
148 delete SKIPPED[key];
149 }
150 }
151 for (key in RESULTS){
152 r = RESULTS[key];
153 if (r == "p"){
154 ++ passed;
155 }else if (r == "f"){
156 ++ failed;
157 fail[failidx++] = key;
158 }else if (r == "e"){
159 ++ unexped;
160 unexp[unexpidx++] = key;
161 }else if (r == "s"){
162 ++ skipped;
163 curdir = DIRS[key];
164 if (curdir in SKIPPED){
165 if (SKIPPED[curdir]){
166 SKIPPED[curdir] = 0;
167 skips[skipidx++] = curdir;
168 }
169 }else{
170 skips[skipidx++] = key;
171 }
172 }else if (r == "n"){
173 ++ ignored;
174 }
175 }
176 printf("\n");
177 if (skipped != 0){
178 printf("\nList of skipped tests:\n");
179 for (i=0; i < skipidx; i++) printf(" %s\n", skips[i]);
180 }
181 if (empty != 0){
182 printf("\nList of directories returning no results:\n");
183 for (i=0; i < empty; i++) printf(" %s\n", blanks[i]);
184 }
185 if (failed != 0){
186 printf("\nList of failed tests:\n");
187 for (i=0; i < failed; i++) printf(" %s\n", fail[i]);
188 }
189 if (unexped != 0){
190 printf("\nList of unexpected errors:\n");
191 for (i=0; i < unexped; i++) printf(" %s\n", unexp[i]);
192 }
193 printf("\n");
194 printf("Summary:\n");
195 printf(" %3d tests passed\n", passed);
196 printf(" %3d tests skipped\n", skipped);
197 printf(" %3d tests failed\n", failed);
198 printf(" %3d tests not started (parent test skipped or failed)\n",
199 ignored);
200 printf(" %3d unexpected errors\n", unexped);
201 printf(" %3d tests considered", nresults);
202 if (nresults != passed + skipped + ignored + failed + unexped){
203 printf (" (totals don't add up??)");
204 }
205 printf ("\n");
206 if (reran != 0){
207 printf(" %3d test dir re-runs\n", reran);
208 }
209 if (failed || unexped){
210 printf("#### Something failed. Exiting with error status.\n\n");
211 exit 4;
212 }
213 }else{
214 for (key in RESULTS){
215 if (RESULTS[key] == "f" || RESULTS[key] == "e"){
216 key = DIRS[key];
217 if (!(key in RERUNS)){
218 RERUNS[key] = 1;
219 if (RERAN[key] < max_retries){
220 printf("%s\n", key);
221 }
222 }
223 }
224 }
225 }
226 }
227 }
228