GCC Code Coverage Report

Line	Branch	Exec	Source
1			// Copyright (c) 2021-2025 ChilliBits. All rights reserved.
2
3			// GCOV_EXCL_START
4
5			#include <string>
6			#include <vector>
7
8			#include <gtest/gtest.h>
9
10			#include <llvm/TargetParser/Host.h>
11			#include <llvm/TargetParser/Triple.h>
12			#include <SourceFile.h>
13			#include <driver/Driver.h>
14			#include <exception/CompilerError.h>
15			#include <exception/LexerError.h>
16			#include <exception/LinkerError.h>
17			#include <exception/ParserError.h>
18			#include <exception/SemanticError.h>
19			#include <global/GlobalResourceManager.h>
20			#include <global/TypeRegistry.h>
21			#include <symboltablebuilder/SymbolTable.h>
22			#include <util/FileUtil.h>
23
24			#include "driver/Driver.h"
25			#include "util/TestUtil.h"
26
27			using namespace spice::compiler;
28
29			namespace spice::testing {
30
31			extern TestDriverCliOptions testDriverCliOptions;
32
33		−	void execTestCase(const TestCase &testCase) {
34			// Check if test is disabled
35		−	if (TestUtil::isDisabled(testCase))
36		−	GTEST_SKIP();
37
38			// Create fake cli options
39		−	const std::filesystem::path sourceFilePath = testCase.testPath / REF_NAME_SOURCE;
40		−	const llvm::Triple targetTriple(llvm::Triple::normalize(llvm::sys::getDefaultTargetTriple()));
41		−	CliOptions cliOptions = {
42			/* mainSourceFile= */ sourceFilePath,
43		−	/* targetTriple= */ targetTriple.getTriple(),
44		−	/* targetArch= */ std::string(targetTriple.getArchName()),
45		−	/* targetVendor= */ std::string(targetTriple.getVendorName()),
46		−	/* targetOs= */ std::string(targetTriple.getOSName()),
47			/* isNativeTarget= */ true,
48			/* useCPUFeatures*/ false, // Disabled because it makes the refs differ on different machines
49			/* execute= */ false, // If we set this to 'true', the compiler will not emit object files
50			/* cacheDir= */ "./cache",
51			/* outputDir= */ "./",
52			/* outputPath= */ "",
53			/* buildMode= */ DEBUG,
54			/* compileJobCount= */ 0,
55			/* ignoreCache */ true,
56		−	/* llvmArgs= */ "",
57			/* printDebugOutput= */ false,
58			CliOptions::DumpSettings{
59			/* dumpCST= */ false,
60			/* dumpAST= */ false,
61			/* dumpSymbolTables= */ false,
62			/* dumpTypes= */ false,
63			/* dumpCacheStats= */ false,
64			/* dumpDependencyGraph= */ false,
65			/* dumpIR= */ false,
66			/* dumpAssembly= */ false,
67			/* dumpObjectFile= */ false,
68			/* dumpToFiles= */ false,
69			/* abortAfterDump */ false,
70			},
71			/* namesForIRValues= */ true,
72			/* useLifetimeMarkers= */ false,
73			/* optLevel= */ O0,
74		−	/* useLTO= */ exists(testCase.testPath / CTL_LTO),
75		−	/* noEntryFct= */ exists(testCase.testPath / CTL_RUN_BUILTIN_TESTS),
76		−	/* generateTestMain= */ exists(testCase.testPath / CTL_RUN_BUILTIN_TESTS),
77			/* staticLinking= */ false,
78		−	/* debugInfo= */ exists(testCase.testPath / CTL_DEBUG_INFO),
79			/* disableVerifier= */ false,
80			/* testMode= */ true,
81			/* comparableOutput= */ true,
82		−	};
83			static_assert(sizeof(CliOptions::DumpSettings) == 11, "CliOptions::DumpSettings struct size changed");
84			static_assert(sizeof(CliOptions) == 360, "CliOptions struct size changed");
85
86			// Instantiate GlobalResourceManager
87		−	GlobalResourceManager resourceManager(cliOptions);
88
89			try {
90			// Create source file instance for main source file
91		−	SourceFile *mainSourceFile = resourceManager.createSourceFile(nullptr, MAIN_FILE_NAME, cliOptions.mainSourceFile, false);
92
93			// Run Lexer and Parser
94		−	mainSourceFile->runLexer();
95		−	mainSourceFile->runParser();
96
97			// Check CST
98		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_PARSE_TREE, [&] {
99		−	mainSourceFile->runCSTVisualizer();
100		−	return mainSourceFile->compilerOutput.cstString;
101			});
102
103			// Build and optimize AST
104		−	mainSourceFile->runASTBuilder();
105
106			// Check AST
107		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_SYNTAX_TREE, [&] {
108		−	mainSourceFile->runASTVisualizer();
109		−	return mainSourceFile->compilerOutput.astString;
110			});
111
112			// Execute import collector and semantic analysis stages
113		−	mainSourceFile->runImportCollector();
114		−	mainSourceFile->runSymbolTableBuilder();
115		−	mainSourceFile->runMiddleEnd(); // TypeChecker pre + post
116
117			// Check symbol table output (check happens here to include updates from type checker)
118		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_SYMBOL_TABLE,
119		−	[&] { return mainSourceFile->globalScope->getSymbolTableJSON().dump(/*indent=*/2); });
120
121			// Fail if an error was expected
122		−	if (exists(testCase.testPath / REF_NAME_ERROR_OUTPUT))
123		−	FAIL() << "Expected error, but got no error";
124
125			// Check dependency graph
126		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_DEP_GRAPH, [&] {
127		−	mainSourceFile->runDependencyGraphVisualizer();
128		−	return mainSourceFile->compilerOutput.depGraphString;
129			});
130
131			// Run backend for all dependencies
132		−	for (SourceFile *sourceFile : mainSourceFile->dependencies | std::views::values)
133		−	sourceFile->runBackEnd();
134
135			// Execute IR generator in normal or debug mode
136		−	mainSourceFile->runIRGenerator();
137
138			// Check unoptimized IR code
139		−	TestUtil::checkRefMatch(
140		−	testCase.testPath / REF_NAME_IR, [&] { return mainSourceFile->compilerOutput.irString; },
141		−	[&](std::string &expectedOutput, std::string &actualOutput) {
142		−	if (cliOptions.generateDebugInfo) {
143			// Remove the lines, containing paths on the local file system
144		−	TestUtil::eraseLinesBySubstring(expectedOutput, " = !DIFile(filename:");
145		−	TestUtil::eraseLinesBySubstring(actualOutput, " = !DIFile(filename:");
146			}
147		−	});
148
149			// Check optimized IR code
150		−	for (uint8_t i = 1; i <= 5; i++) {
151		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_OPT_IR[i - 1], [&] {
152		−	cliOptions.optLevel = static_cast<OptLevel>(i);
153
154		−	if (cliOptions.useLTO) {
155		−	mainSourceFile->runPreLinkIROptimizer();
156		−	mainSourceFile->runBitcodeLinker();
157		−	mainSourceFile->runPostLinkIROptimizer();
158			} else {
159		−	mainSourceFile->runDefaultIROptimizer();
160			}
161
162		−	return mainSourceFile->compilerOutput.irOptString;
163			});
164			}
165
166			// Link the bitcode if not happened yet
167		−	if (cliOptions.useLTO && cliOptions.optLevel == O0)
168		−	mainSourceFile->runBitcodeLinker();
169
170			// Check assembly code (only when not running test on GitHub Actions)
171		−	bool objectFilesEmitted = false;
172		−	if (!testDriverCliOptions.isGitHubActions) {
173		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_ASM, [&] {
174		−	mainSourceFile->runObjectEmitter();
175		−	objectFilesEmitted = true;
176
177		−	return mainSourceFile->compilerOutput.asmString;
178			});
179			}
180
181			// Check warnings
182		−	mainSourceFile->collectAndPrintWarnings();
183		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_WARNING_OUTPUT, [&] {
184		−	std::stringstream actualWarningString;
185		−	for (const CompilerWarning &warning : mainSourceFile->compilerOutput.warnings)
186		−	actualWarningString << warning.warningMessage << "\n";
187		−	return actualWarningString.str();
188		−	});
189
190			// Do linking and conclude compilation
191		−	const bool needsNormalRun = TestUtil::doesRefExist(testCase.testPath / REF_NAME_EXECUTION_OUTPUT);
192		−	const bool needsDebuggerRun = TestUtil::doesRefExist(testCase.testPath / REF_NAME_GDB_OUTPUT);
193		−	if (needsNormalRun || needsDebuggerRun) {
194			// Prepare linker
195		−	resourceManager.linker.outputPath = TestUtil::getDefaultExecutableName();
196
197			// Parse linker flags
198		−	const std::filesystem::path linkerFlagsFile = testCase.testPath / INPUT_NAME_LINKER_FLAGS;
199		−	if (exists(linkerFlagsFile))
200		−	for (const std::string &linkerFlag : TestUtil::getFileContentLinesVector(linkerFlagsFile))
201		−	resourceManager.linker.addLinkerFlag(linkerFlag);
202
203			// Emit main source file object if not done already
204		−	if (!objectFilesEmitted)
205		−	mainSourceFile->runObjectEmitter();
206
207			// Conclude the compilation
208		−	mainSourceFile->concludeCompilation();
209
210			// Prepare and run linker
211		−	resourceManager.linker.prepare();
212		−	resourceManager.linker.link();
213		−	}
214
215			// Check type registry output
216		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_TYPE_REGISTRY, [&] { return TypeRegistry::dump(); });
217
218			// Check cache stats output
219		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_CACHE_STATS, [&] {
220		−	std::stringstream cacheStats;
221		−	cacheStats << FunctionManager::dumpLookupCacheStatistics() << std::endl;
222		−	cacheStats << StructManager::dumpLookupCacheStatistics() << std::endl;
223		−	cacheStats << InterfaceManager::dumpLookupCacheStatistics() << std::endl;
224		−	return cacheStats.str();
225		−	});
226
227			// Check if the execution output matches the expected output
228		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_EXECUTION_OUTPUT, [&] {
229		−	const std::filesystem::path cliFlagsFile = testCase.testPath / INPUT_NAME_CLI_FLAGS;
230			// Execute binary
231		−	std::stringstream cmd;
232		−	if (testDriverCliOptions.enableLeakDetection)
233		−	cmd << "valgrind -q --leak-check=full --num-callers=100 --error-exitcode=1 ";
234		−	cmd << TestUtil::getDefaultExecutableName();
235		−	if (exists(cliFlagsFile))
236		−	cmd << " " << TestUtil::getFileContentLinesVector(cliFlagsFile).at(0);
237		−	const auto [output, exitCode] = FileUtil::exec(cmd.str(), true);
238
239			#if not OS_WINDOWS // Windows does not give us the exit code, so we cannot check it on Windows
240			// Check if the exit code matches the expected one
241			// If no exit code ref file exists, check against 0
242		−	if (TestUtil::checkRefMatch(testCase.testPath / REF_NAME_EXIT_CODE, [&] { return std::to_string(exitCode); })) {
243		−	EXPECT_NE(0, exitCode) << "Program exited with zero exit code, but expected erronous exit code";
244			} else {
245		−	EXPECT_EQ(0, exitCode) << "Program exited with non-zero exit code";
246			}
247			#endif
248
249		−	return output;
250		−	});
251
252			// Check if the debugger output matches the expected output
253		−	if (!testDriverCliOptions.isGitHubActions) { // GDB tests are currently not support on GH actions
254		−	TestUtil::checkRefMatch(
255		−	testCase.testPath / REF_NAME_GDB_OUTPUT,
256		−	[&] {
257			// Execute debugger script
258		−	std::filesystem::path gdbScriptPath = testCase.testPath / CTL_DEBUG_SCRIPT;
259		−	EXPECT_TRUE(std::filesystem::exists(gdbScriptPath)) << "Debug output requested, but debug script not found";
260		−	gdbScriptPath.make_preferred();
261		−	const std::string cmd = "gdb -x " + gdbScriptPath.string() + " " + TestUtil::getDefaultExecutableName();
262		−	const auto [output, exitCode] = FileUtil::exec(cmd);
263
264			#if not OS_WINDOWS // Windows does not give us the exit code, so we cannot check it on Windows
265		−	EXPECT_EQ(0, exitCode) << "GDB exited with non-zero exit code when running debug script";
266			#endif
267
268		−	return output;
269		−	},
270		−	[&](std::string &expectedOutput, std::string &actualOutput) {
271			// Do not compare against the GDB header
272		−	TestUtil::eraseGDBHeader(expectedOutput);
273		−	TestUtil::eraseGDBHeader(actualOutput);
274		−	});
275			}
276		−	} catch (LexerError &error) {
277		−	TestUtil::handleError(testCase, error);
278		−	} catch (ParserError &error) {
279		−	TestUtil::handleError(testCase, error);
280		−	} catch (SemanticError &error) {
281		−	TestUtil::handleError(testCase, error);
282		−	} catch (CompilerError &error) {
283		−	TestUtil::handleError(testCase, error);
284		−	} catch (LinkerError &error) {
285		−	TestUtil::handleError(testCase, error);
286		−	} catch (std::exception &error) {
287		−	TestUtil::handleError(testCase, error);
288		−	}
289
290		−	SUCCEED();
291		−	}
292
293			class CommonTests : public ::testing::TestWithParam<TestCase> {};
294		−	TEST_P(CommonTests, ) { execTestCase(GetParam()); }
295		−	INSTANTIATE_TEST_SUITE_P(, CommonTests, ::testing::ValuesIn(TestUtil::collectTestCases("common", false)),
296			TestUtil::NameResolver());
297
298			class LexerTests : public ::testing::TestWithParam<TestCase> {};
299		−	TEST_P(LexerTests, ) { execTestCase(GetParam()); }
300		−	INSTANTIATE_TEST_SUITE_P(, LexerTests, ::testing::ValuesIn(TestUtil::collectTestCases("lexer", false)), TestUtil::NameResolver());
301
302			class ParserTests : public ::testing::TestWithParam<TestCase> {};
303		−	TEST_P(ParserTests, ) { execTestCase(GetParam()); }
304		−	INSTANTIATE_TEST_SUITE_P(, ParserTests, ::testing::ValuesIn(TestUtil::collectTestCases("parser", false)),
305			TestUtil::NameResolver());
306
307			class SymbolTableBuilderTests : public ::testing::TestWithParam<TestCase> {};
308		−	TEST_P(SymbolTableBuilderTests, ) { execTestCase(GetParam()); }
309		−	INSTANTIATE_TEST_SUITE_P(, SymbolTableBuilderTests, ::testing::ValuesIn(TestUtil::collectTestCases("symboltablebuilder", true)),
310			TestUtil::NameResolver());
311
312			class TypeCheckerTests : public ::testing::TestWithParam<TestCase> {};
313		−	TEST_P(TypeCheckerTests, ) { execTestCase(GetParam()); }
314		−	INSTANTIATE_TEST_SUITE_P(, TypeCheckerTests, ::testing::ValuesIn(TestUtil::collectTestCases("typechecker", true)),
315			TestUtil::NameResolver());
316
317			class IRGeneratorTests : public ::testing::TestWithParam<TestCase> {};
318		−	TEST_P(IRGeneratorTests, ) { execTestCase(GetParam()); }
319		−	INSTANTIATE_TEST_SUITE_P(, IRGeneratorTests, ::testing::ValuesIn(TestUtil::collectTestCases("irgenerator", true)),
320			TestUtil::NameResolver());
321
322			class StdTests : public ::testing::TestWithParam<TestCase> {};
323		−	TEST_P(StdTests, ) { execTestCase(GetParam()); }
324		−	INSTANTIATE_TEST_SUITE_P(, StdTests, ::testing::ValuesIn(TestUtil::collectTestCases("std", true)), TestUtil::NameResolver());
325
326			class BenchmarkTests : public ::testing::TestWithParam<TestCase> {};
327		−	TEST_P(BenchmarkTests, ) { execTestCase(GetParam()); }
328		−	INSTANTIATE_TEST_SUITE_P(, BenchmarkTests, ::testing::ValuesIn(TestUtil::collectTestCases("benchmark", false)),
329			TestUtil::NameResolver());
330
331			class ExampleTests : public ::testing::TestWithParam<TestCase> {};
332		−	TEST_P(ExampleTests, ) { execTestCase(GetParam()); }
333		−	INSTANTIATE_TEST_SUITE_P(, ExampleTests, ::testing::ValuesIn(TestUtil::collectTestCases("examples", false)),
334			TestUtil::NameResolver());
335
336			class BootstrapCompilerTests : public ::testing::TestWithParam<TestCase> {};
337		−	TEST_P(BootstrapCompilerTests, ) { execTestCase(GetParam()); }
338		−	INSTANTIATE_TEST_SUITE_P(, BootstrapCompilerTests, ::testing::ValuesIn(TestUtil::collectTestCases("bootstrap-compiler", false)),
339			TestUtil::NameResolver());
340
341			} // namespace spice::testing
342
343			// GCOV_EXCL_STOP
344