1 /// Copyright: Copyright (c) 2017-2019 Andrey Penechko.
2 /// License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0).
3 /// Authors: Andrey Penechko.
4 module tester;
5
6 import vox.all;
7 import tests.aggregates;
8 import tests.ctfe;
9 import tests.passing;
10 import tests.failing;
11 import tests.exe;
12 import tests.reg_alloc;
13 public import vox.all : HostSymbol, Slice, SliceString, TargetOs;
14
15 public import std.format : formattedWrite, format;
16 import std.string : stripLeft, strip;
17
18 // test entry point when compiling with -unittest or dub test
19 unittest
20 {
21 import tester;
22 import tests.amd64asm;
23
24 int numFailedTests = 0;
25 numFailedTests += runAllTests(StopOnFirstFail.no);
26 numFailedTests += testAmd64Asm();
27
28 assert(numFailedTests == 0);
29 }
30
31 int runDevTests()
32 {
33 Test test = makeTest!(test227_1);
34
35 Driver driver;
36 driver.initialize(jitPasses);
37 driver.context.buildType = BuildType.jit;
38 driver.context.validateIr = true;
39 driver.context.printTraceOnError = true;
40 driver.context.printTodos = true;
41 //driver.context.disableDCE = true;
42 //driver.context.runTesters = false;
43 //driver.context.debugRegAlloc = true;
44 //driver.context.buildType = BuildType.exe;
45 //driver.passes = exePasses;
46 //driver.context.windowsSubsystem = WindowsSubsystem.WINDOWS_GUI;
47
48 //test.targetOs = TargetOs.linux;
49 //test.printFilter = "i32_to_str";
50
51 scope(exit) {
52 //driver.context.printMemSize;
53 driver.releaseMemory;
54 }
55
56 FuncDumpSettings dumpSettings;
57 dumpSettings.printBlockFlags = true;
58 //dumpSettings.printBlockOuts = true;
59
60 //driver.context.printSource = true;
61 //driver.context.printLexemes = true;
62 //driver.context.printAstFresh = true;
63 //driver.context.printAstSema = true;
64 //driver.context.printIr = true;
65 //driver.context.printIrOptEach = true;
66 //driver.context.printIrOpt = true;
67 //driver.context.printIrLowerEach = true;
68 //driver.context.printIrLower = true;
69 //driver.context.printLir = true;
70 //driver.context.printLiveIntervals = true;
71 //driver.context.printLirRA = true;
72 //driver.context.printStackLayout = true;
73 //driver.context.printStaticData = true;
74 //driver.context.printCodeHex = true;
75 //driver.context.printSymbols = true;
76 //driver.context.printTimings = true;
77
78 TestResult res = tryRunSingleTest(driver, dumpSettings, DumpTest.yes, test);
79 //writefln("%s", driver.context.numCtfeRuns);
80
81 return cast(int)(res == TestResult.failure);
82 }
83
84 enum StopOnFirstFail : bool { no = false, yes = true }
85
86 // Returns number of failed tests
87 int runAllTests(StopOnFirstFail stopOnFirstFail)
88 {
89 auto startInitTime = currTime;
90 Driver driver;
91 driver.initialize(jitPasses);
92 driver.context.buildType = BuildType.jit;
93 driver.context.buildDebug = false;
94 driver.context.validateIr = true;
95 //driver.context.debugRegAlloc = true;
96 //driver.context.printIr = true;
97
98 //driver.context.printAstSema = true;
99 // Is slow when doing failing tests
100 //driver.context.printTraceOnError = true;
101 auto endInitTime = currTime;
102
103 FuncDumpSettings dumpSettings;
104 dumpSettings.printBlockFlags = true;
105
106 Test[] jitTests =
107 tests.passing.passingTests ~
108 tests.aggregates.aggregatesTests ~
109 tests.ctfe.ctfeTests ~
110 tests.failing.failingTests;
111 Test[] regAllocTests = tests.reg_alloc.regAllocTests;
112 Test[] exeTests = tests.exe.exeTests;
113
114 size_t numTests = jitTests.length + regAllocTests.length;
115 version(Windows) numTests += exeTests.length;
116 size_t numSuccessfulTests;
117 writefln("Running %s tests", numTests);
118
119 size_t indexOffset = 0;
120 bool failed = false;
121
122 void runTests(Test[] tests)
123 {
124 foreach(size_t i, ref Test test; tests)
125 {
126 if (stopOnFirstFail && failed) break;
127
128 TestResult res = tryRunSingleTest(driver, dumpSettings, DumpTest.no, test);
129
130 if (res == TestResult.failure)
131 {
132 writefln("%s/%s test `%s` %s", indexOffset+i+1, numTests, test.testName, res);
133 failed = true;
134 if (stopOnFirstFail) writeln("Stopping on first fail");
135 writeln;
136 }
137 else
138 {
139 //writefln("%s `%s` success", indexOffset+i+1, test.testName);
140 ++numSuccessfulTests;
141 }
142 }
143 indexOffset += tests.length;
144 }
145
146 auto time1 = currTime;
147
148 runTests(jitTests);
149 auto time2 = currTime;
150
151 driver.context.debugRegAlloc = true;
152 runTests(regAllocTests);
153 driver.context.debugRegAlloc = false;
154 auto time3 = currTime;
155
156 driver.context.buildType = BuildType.exe;
157 driver.passes = exePasses;
158 version(Windows) runTests(exeTests);
159 auto time4 = currTime;
160
161 Duration durationJit = time2-time1;
162 Duration durationRA = time3-time2;
163 Duration durationExe = time4-time3;
164 Duration durationTotal = time4-time1;
165
166 auto startReleaseTime = currTime;
167 driver.releaseMemory;
168 auto endReleaseTime = currTime;
169
170 writefln("jit(%s tests) %ss",
171 jitTests.length,
172 scaledNumberFmt(durationJit),
173 );
174 writefln("RA(%s tests) %ss",
175 regAllocTests.length,
176 scaledNumberFmt(durationRA),
177 );
178 writefln("exe(%s tests) %ss",
179 exeTests.length,
180 scaledNumberFmt(durationExe),
181 );
182 writefln("Done %s/%s successful in %ss, init %ss, release %ss",
183 numSuccessfulTests,
184 numTests,
185 scaledNumberFmt(durationTotal),
186 scaledNumberFmt(endInitTime-startInitTime),
187 scaledNumberFmt(endReleaseTime-startReleaseTime),
188 );
189
190 return cast(int)(numTests - numSuccessfulTests);
191 }
192
193 enum DumpTest : bool { no = false, yes = true }
194 enum TestResult { failure, success }
195 struct Test
196 {
197 string testName;
198 string harData;
199 void function(ref TestContext) tester;
200 HostSymbol[] hostSymbols;
201 TargetOs targetOs = CompilationContext.hostOS;
202 string printFilter;
203 }
204
205 Test makeTest(alias test)() {
206 static assert (__traits(getAttributes, test).length > 0);
207 TestInfo info;
208 TargetOs target;
209 foreach (attr; __traits(getAttributes, test)) {
210 static if (is (typeof(attr) == TestInfo)) {
211 info = attr;
212 } else static if (is (typeof(attr) == TargetOs)) {
213 target = attr;
214 }
215 }
216 return Test(__traits(identifier, test), test, info.tester, info.hostSymbols, target);
217 }
218
219 Test[] collectTests(alias M)()
220 {
221 Test[] tests;
222 import std.traits : hasUDA;
223 foreach(m; __traits(allMembers, M))
224 {
225 alias member = __traits(getMember, M, m);
226 static if (hasUDA!(member, TestInfo)) {
227 static if (is(typeof(member) == immutable string)) {
228 tests ~= makeTest!member;
229 }
230 }
231 }
232 return tests;
233 }
234
235 struct TestInfo
236 {
237 void function(ref TestContext) tester;
238 HostSymbol[] hostSymbols;
239 }
240
241 struct TestContext
242 {
243 Driver* driver;
244
245 auto getFunctionPtr(ResultType, ParamTypes...)(string funcName) {
246 Identifier funcId = driver.context.idMap.getOrRegFqn(&driver.context, funcName);
247 FunctionDeclNode* func = driver.context.findUniquelyNamedFunction(funcId);
248 return driver.context.getFunctionPtr!(ResultType, ParamTypes)(func);
249 }
250 }
251
252 /// Global test output for jit tests
253 /// Cleared automatically after each test
254 TextSink testSink;
255
256 extern(C) void external_noop() {}
257
258 TestResult tryRunSingleTest(ref Driver driver, ref FuncDumpSettings dumpSettings, DumpTest dumpTest, Test curTest)
259 {
260 scope(exit) testSink.clear;
261 try
262 {
263 TestResult res = runSingleTest(driver, dumpSettings, dumpTest, curTest);
264 return res;
265 }
266 catch(CompilationException e) {
267 writeln(driver.context.sink.text);
268 if (e.isICE)
269 writeln(e);
270 return TestResult.failure;
271 }
272 catch(Throwable t) {
273 writeln(driver.context.sink.text);
274 writeln(t);
275 return TestResult.failure;
276 }
277 }
278
279 TestResult runSingleTest(ref Driver driver, ref FuncDumpSettings dumpSettings, DumpTest dumpTest, Test curTest)
280 {
281 import std.string : lineSplitter;
282 import std.algorithm : joiner, equal;
283 bool isFailingTest;
284 string expectedError;
285
286 enum NUM_ITERS = 1;
287 auto time1 = currTime;
288
289 // setup modules
290 driver.beginCompilation();
291
292 if (curTest.printFilter.length) driver.context.setDumpFilter(curTest.printFilter);
293
294 driver.context.targetOs = curTest.targetOs;
295
296 driver.addHostSymbols(curTest.hostSymbols);
297
298 void onHarFile(SourceFileInfo fileInfo)
299 {
300 if (fileInfo.name == "<error>") {
301 assert(!isFailingTest, format("Multiple <error> files in test `%s`", curTest.testName));
302 isFailingTest = true;
303 expectedError = cast(string)fileInfo.content.strip;
304 } else {
305 driver.addModule(fileInfo);
306 }
307 }
308
309 // We strip beginning to allow newline between `q{` and `--- <filename>` for better readability
310 string strippedHar = curTest.harData.stripLeft;
311 parseHar(driver.context, "test.har", strippedHar, &onHarFile);
312
313 // Splitting lines gets rid of \r\n on windows, which will cause mismatch with error message
314 auto expectedErrorRange = expectedError.lineSplitter.joiner("\n");
315
316 // compile
317 try
318 {
319 driver.compile();
320 }
321 catch(CompilationException e)
322 {
323 if (!isFailingTest) throw e;
324 if (e.isICE) throw e;
325
326 // successfully matched the error message(s)
327 if (equal(driver.context.errorSink.text, expectedErrorRange)) {
328 return TestResult.success;
329 }
330
331 writefln("Test `%s` failed", curTest.testName);
332 writefln("Expected error:");
333 writeln(expectedErrorRange);
334 writefln("Received error:");
335 writeln(driver.context.errorSink.text);
336 writefln("Stack trace:");
337 writeln(driver.context.sink.text);
338 writeln(e.info);
339 return TestResult.failure;
340 }
341
342 // Compiled successfully, but expected to fail
343 if (isFailingTest) {
344 writefln("Test `%s` compiled successfully, but expected to fail", curTest.testName);
345 writefln("Expected error:");
346 writeln(expectedErrorRange);
347 return TestResult.failure;
348 }
349
350 // finalize code pages
351 driver.markCodeAsExecutable();
352
353 auto time2 = currTime;
354 if (dumpTest && driver.context.printTimings)
355 writefln("Compiled in %ss", scaledNumberFmt(time2-time1));
356
357 if (!driver.context.runTesters) return TestResult.success;
358 if (driver.context.targetOs != driver.context.hostOS) return TestResult.success;
359
360 final switch (driver.context.buildType)
361 {
362 case BuildType.jit:
363 if (curTest.tester) {
364 auto testContext = TestContext(&driver);
365 if (dumpTest) writefln("Running: %s tester", curTest.testName);
366 auto time3 = currTime;
367 curTest.tester(testContext);
368 auto time4 = currTime;
369 if (dumpTest && driver.context.printTimings)
370 writefln("Run in %ss", scaledNumberFmt(time4-time3));
371 }
372 break;
373
374 case BuildType.exe:
375 import std.process;
376 import std.file : exists;
377 import std.path;
378 if(exists(driver.context.outputFilename))
379 {
380 if (dumpTest) writef("Running: %s", driver.context.outputFilename.absolutePath);
381 auto time3 = currTime;
382 auto result = execute(driver.context.outputFilename);
383 auto time4 = currTime;
384 if (dumpTest && driver.context.printTimings)
385 writefln("Run in %ss", scaledNumberFmt(time4-time3));
386 if (dumpTest) writefln(", status %s, output '%s'", result.status, result.output);
387 }
388 else
389 {
390 writefln("No executable produced '%s'", driver.context.outputFilename);
391 }
392 break;
393 }
394
395 return TestResult.success;
396 }