GCC Code Coverage Report

Line	Branch	Exec	Source
1			// Copyright (c) 2021-2025 ChilliBits. All rights reserved.
2
3			// GCOV_EXCL_START
4
5			#include <string>
6			#include <vector>
7
8			#include <gtest/gtest.h>
9
10			#include <llvm/TargetParser/Host.h>
11			#include <llvm/TargetParser/Triple.h>
12
13			#include <SourceFile.h>
14			#include <driver/Driver.h>
15			#include <exception/CompilerError.h>
16			#include <exception/LexerError.h>
17			#include <exception/LinkerError.h>
18			#include <exception/ParserError.h>
19			#include <exception/SemanticError.h>
20			#include <global/GlobalResourceManager.h>
21			#include <global/TypeRegistry.h>
22			#include <symboltablebuilder/SymbolTable.h>
23			#include <util/FileUtil.h>
24
25			#include "util/TestUtil.h"
26
27			using namespace spice::compiler;
28
29			namespace spice::testing {
30
31		−	void execTestCase(const TestCase &testCase) {
32			// Check if test is disabled
33		−	if (TestUtil::isDisabled(testCase, skipNonGitHubTests))
34		−	GTEST_SKIP();
35
36			// Create fake cli options
37		−	const std::filesystem::path sourceFilePath = testCase.testPath / REF_NAME_SOURCE;
38		−	const llvm::Triple targetTriple(llvm::Triple::normalize(llvm::sys::getDefaultTargetTriple()));
39		−	CliOptions cliOptions = {
40			/* mainSourceFile= */ sourceFilePath,
41		−	/* targetTriple= */ targetTriple.getTriple(),
42		−	/* targetArch= */ std::string(targetTriple.getArchName()),
43		−	/* targetVendor= */ std::string(targetTriple.getVendorName()),
44		−	/* targetOs= */ std::string(targetTriple.getOSName()),
45			/* isNativeTarget= */ true,
46			/* useCPUFeatures*/ false, // Disabled because it makes the refs differ on different machines
47			/* execute= */ false, // If we set this to 'true', the compiler will not emit object files
48			/* cacheDir= */ "./cache",
49			/* outputDir= */ "./",
50			/* outputPath= */ "",
51			/* buildMode= */ DEBUG,
52			/* compileJobCount= */ 0,
53			/* ignoreCache */ true,
54		−	/* llvmArgs= */ "",
55			/* printDebugOutput= */ false,
56			CliOptions::DumpSettings{
57			/* dumpCST= */ false,
58			/* dumpAST= */ false,
59			/* dumpSymbolTables= */ false,
60			/* dumpTypes= */ false,
61			/* dumpDependencyGraph= */ false,
62			/* dumpIR= */ false,
63			/* dumpAssembly= */ false,
64			/* dumpObjectFile= */ false,
65			/* dumpToFiles= */ false,
66			/* abortAfterDump */ false,
67			},
68			/* namesForIRValues= */ true,
69			/* useLifetimeMarkers= */ false,
70			/* optLevel= */ O0,
71		−	/* useLTO= */ exists(testCase.testPath / CTL_LTO),
72		−	/* noEntryFct= */ exists(testCase.testPath / CTL_RUN_BUILTIN_TESTS),
73		−	/* generateTestMain= */ exists(testCase.testPath / CTL_RUN_BUILTIN_TESTS),
74			/* staticLinking= */ false,
75		−	/* debugInfo= */ exists(testCase.testPath / CTL_DEBUG_INFO),
76			/* disableVerifier= */ false,
77			/* testMode= */ true,
78		−	};
79			static_assert(sizeof(CliOptions::DumpSettings) == 10, "CliOptions::DumpSettings struct size changed");
80			static_assert(sizeof(CliOptions) == 360, "CliOptions struct size changed");
81
82			// Instantiate GlobalResourceManager
83		−	GlobalResourceManager resourceManager(cliOptions);
84
85			try {
86			// Create source file instance for main source file
87		−	SourceFile *mainSourceFile = resourceManager.createSourceFile(nullptr, MAIN_FILE_NAME, cliOptions.mainSourceFile, false);
88
89			// Run Lexer and Parser
90		−	mainSourceFile->runLexer();
91		−	mainSourceFile->runParser();
92
93			// Check CST
94		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_PARSE_TREE, [&] {
95		−	mainSourceFile->runCSTVisualizer();
96		−	return mainSourceFile->compilerOutput.cstString;
97			});
98
99			// Build and optimize AST
100		−	mainSourceFile->runASTBuilder();
101
102			// Check AST
103		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_SYNTAX_TREE, [&] {
104		−	mainSourceFile->runASTVisualizer();
105		−	return mainSourceFile->compilerOutput.astString;
106			});
107
108			// Execute import collector and semantic analysis stages
109		−	mainSourceFile->runImportCollector();
110		−	mainSourceFile->runSymbolTableBuilder();
111		−	mainSourceFile->runMiddleEnd(); // TypeChecker pre + post
112
113			// Check symbol table output (check happens here to include updates from type checker)
114		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_SYMBOL_TABLE,
115		−	[&] { return mainSourceFile->globalScope->getSymbolTableJSON().dump(/*indent=*/2); });
116
117			// Fail if an error was expected
118		−	if (exists(testCase.testPath / REF_NAME_ERROR_OUTPUT))
119		−	FAIL() << "Expected error, but got no error";
120
121			// Check dependency graph
122		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_DEP_GRAPH, [&] {
123		−	mainSourceFile->runDependencyGraphVisualizer();
124		−	return mainSourceFile->compilerOutput.depGraphString;
125			});
126
127			// Run backend for all dependencies
128		−	for (SourceFile *sourceFile : mainSourceFile->dependencies | std::views::values)
129		−	sourceFile->runBackEnd();
130
131			// Execute IR generator in normal or debug mode
132		−	mainSourceFile->runIRGenerator();
133
134			// Check unoptimized IR code
135		−	TestUtil::checkRefMatch(
136		−	testCase.testPath / REF_NAME_IR, [&] { return mainSourceFile->compilerOutput.irString; },
137		−	[&](std::string &expectedOutput, std::string &actualOutput) {
138		−	if (cliOptions.generateDebugInfo) {
139			// Remove the lines, containing paths on the local file system
140		−	TestUtil::eraseLinesBySubstring(expectedOutput, " = !DIFile(filename:");
141		−	TestUtil::eraseLinesBySubstring(actualOutput, " = !DIFile(filename:");
142			}
143		−	});
144
145			// Check optimized IR code
146		−	for (uint8_t i = 1; i <= 5; i++) {
147		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_OPT_IR[i - 1], [&] {
148		−	cliOptions.optLevel = static_cast<OptLevel>(i);
149
150		−	if (cliOptions.useLTO) {
151		−	mainSourceFile->runPreLinkIROptimizer();
152		−	mainSourceFile->runBitcodeLinker();
153		−	mainSourceFile->runPostLinkIROptimizer();
154			} else {
155		−	mainSourceFile->runDefaultIROptimizer();
156			}
157
158		−	return mainSourceFile->compilerOutput.irOptString;
159			});
160			}
161
162			// Link the bitcode if not happened yet
163		−	if (cliOptions.useLTO && cliOptions.optLevel == O0)
164		−	mainSourceFile->runBitcodeLinker();
165
166			// Check assembly code
167		−	bool objectFilesEmitted = false;
168		−	if (!skipNonGitHubTests) {
169		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_ASM, [&] {
170		−	mainSourceFile->runObjectEmitter();
171		−	objectFilesEmitted = true;
172
173		−	return mainSourceFile->compilerOutput.asmString;
174			});
175			}
176
177			// Check warnings
178		−	mainSourceFile->collectAndPrintWarnings();
179		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_WARNING_OUTPUT, [&] {
180		−	std::stringstream actualWarningString;
181		−	for (const CompilerWarning &warning : mainSourceFile->compilerOutput.warnings)
182		−	actualWarningString << warning.warningMessage << "\n";
183		−	return actualWarningString.str();
184		−	});
185
186			// Do linking and conclude compilation
187		−	const bool needsNormalRun = exists(testCase.testPath / REF_NAME_EXECUTION_OUTPUT);
188		−	const bool needsDebuggerRun = exists(testCase.testPath / REF_NAME_GDB_OUTPUT);
189		−	if (needsNormalRun || needsDebuggerRun) {
190			// Prepare linker
191		−	resourceManager.linker.outputPath = TestUtil::getDefaultExecutableName();
192
193			// Parse linker flags
194		−	const std::filesystem::path linkerFlagsFile = testCase.testPath / INPUT_NAME_LINKER_FLAGS;
195		−	if (exists(linkerFlagsFile))
196		−	for (const std::string &linkerFlag : TestUtil::getFileContentLinesVector(linkerFlagsFile))
197		−	resourceManager.linker.addLinkerFlag(linkerFlag);
198
199			// Emit main source file object if not done already
200		−	if (!objectFilesEmitted)
201		−	mainSourceFile->runObjectEmitter();
202
203			// Conclude the compilation
204		−	mainSourceFile->concludeCompilation();
205
206			// Prepare and run linker
207		−	resourceManager.linker.prepare();
208		−	resourceManager.linker.link();
209		−	}
210
211			// Check type registry output
212		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_TYPE_REGISTRY, [&] { return TypeRegistry::dump(); });
213
214			// Check if the execution output matches the expected output
215		−	TestUtil::checkRefMatch(testCase.testPath / REF_NAME_EXECUTION_OUTPUT, [&] {
216		−	const std::filesystem::path cliFlagsFile = testCase.testPath / INPUT_NAME_CLI_FLAGS;
217			// Execute binary
218		−	std::stringstream cmd;
219		−	if (enableLeakDetection)
220		−	cmd << "valgrind -q --leak-check=full --num-callers=100 --error-exitcode=1 ";
221		−	cmd << TestUtil::getDefaultExecutableName();
222		−	if (exists(cliFlagsFile))
223		−	cmd << " " << TestUtil::getFileContentLinesVector(cliFlagsFile).at(0);
224		−	const auto [output, exitCode] = FileUtil::exec(cmd.str(), true);
225
226			#if not OS_WINDOWS // Windows does not give us the exit code, so we cannot check it on Windows
227			// Check if the exit code matches the expected one
228			// If no exit code ref file exists, check against 0
229		−	if (TestUtil::checkRefMatch(testCase.testPath / REF_NAME_EXIT_CODE, [&] { return std::to_string(exitCode); })) {
230		−	EXPECT_NE(0, exitCode) << "Program exited with zero exit code, but expected erronous exit code";
231			} else {
232		−	EXPECT_EQ(0, exitCode) << "Program exited with non-zero exit code";
233			}
234			#endif
235
236		−	return output;
237		−	});
238
239			// Check if the debugger output matches the expected output
240		−	if (!skipNonGitHubTests) { // GDB tests are currently not support on GH actions
241		−	TestUtil::checkRefMatch(
242		−	testCase.testPath / REF_NAME_GDB_OUTPUT,
243		−	[&] {
244			// Execute debugger script
245		−	std::filesystem::path gdbScriptPath = testCase.testPath / CTL_DEBUG_SCRIPT;
246		−	EXPECT_TRUE(std::filesystem::exists(gdbScriptPath)) << "Debug output requested, but debug script not found";
247		−	gdbScriptPath.make_preferred();
248		−	const std::string cmd = "gdb -x " + gdbScriptPath.string() + " " + TestUtil::getDefaultExecutableName();
249		−	const auto [output, exitCode] = FileUtil::exec(cmd);
250
251			#if not OS_WINDOWS // Windows does not give us the exit code, so we cannot check it on Windows
252		−	EXPECT_EQ(0, exitCode) << "GDB exited with non-zero exit code when running debug script";
253			#endif
254
255		−	return output;
256		−	},
257		−	[&](std::string &expectedOutput, std::string &actualOutput) {
258			// Do not compare against the GDB header
259		−	TestUtil::eraseGDBHeader(expectedOutput);
260		−	TestUtil::eraseGDBHeader(actualOutput);
261		−	});
262			}
263		−	} catch (LexerError &error) {
264		−	TestUtil::handleError(testCase, error);
265		−	} catch (ParserError &error) {
266		−	TestUtil::handleError(testCase, error);
267		−	} catch (SemanticError &error) {
268		−	TestUtil::handleError(testCase, error);
269		−	} catch (CompilerError &error) {
270		−	TestUtil::handleError(testCase, error);
271		−	} catch (LinkerError &error) {
272		−	TestUtil::handleError(testCase, error);
273		−	} catch (std::exception &error) {
274		−	TestUtil::handleError(testCase, error);
275		−	}
276
277		−	SUCCEED();
278		−	}
279
280			class CommonTests : public ::testing::TestWithParam<TestCase> {};
281		−	TEST_P(CommonTests, ) { execTestCase(GetParam()); }
282		−	INSTANTIATE_TEST_SUITE_P(, CommonTests, ::testing::ValuesIn(TestUtil::collectTestCases("common", false)),
283			TestUtil::NameResolver());
284
285			class LexerTests : public ::testing::TestWithParam<TestCase> {};
286		−	TEST_P(LexerTests, ) { execTestCase(GetParam()); }
287		−	INSTANTIATE_TEST_SUITE_P(, LexerTests, ::testing::ValuesIn(TestUtil::collectTestCases("lexer", false)), TestUtil::NameResolver());
288
289			class ParserTests : public ::testing::TestWithParam<TestCase> {};
290		−	TEST_P(ParserTests, ) { execTestCase(GetParam()); }
291		−	INSTANTIATE_TEST_SUITE_P(, ParserTests, ::testing::ValuesIn(TestUtil::collectTestCases("parser", false)),
292			TestUtil::NameResolver());
293
294			class SymbolTableBuilderTests : public ::testing::TestWithParam<TestCase> {};
295		−	TEST_P(SymbolTableBuilderTests, ) { execTestCase(GetParam()); }
296		−	INSTANTIATE_TEST_SUITE_P(, SymbolTableBuilderTests, ::testing::ValuesIn(TestUtil::collectTestCases("symboltablebuilder", true)),
297			TestUtil::NameResolver());
298
299			class TypeCheckerTests : public ::testing::TestWithParam<TestCase> {};
300		−	TEST_P(TypeCheckerTests, ) { execTestCase(GetParam()); }
301		−	INSTANTIATE_TEST_SUITE_P(, TypeCheckerTests, ::testing::ValuesIn(TestUtil::collectTestCases("typechecker", true)),
302			TestUtil::NameResolver());
303
304			class IRGeneratorTests : public ::testing::TestWithParam<TestCase> {};
305		−	TEST_P(IRGeneratorTests, ) { execTestCase(GetParam()); }
306		−	INSTANTIATE_TEST_SUITE_P(, IRGeneratorTests, ::testing::ValuesIn(TestUtil::collectTestCases("irgenerator", true)),
307			TestUtil::NameResolver());
308
309			class StdTests : public ::testing::TestWithParam<TestCase> {};
310		−	TEST_P(StdTests, ) { execTestCase(GetParam()); }
311		−	INSTANTIATE_TEST_SUITE_P(, StdTests, ::testing::ValuesIn(TestUtil::collectTestCases("std", true)), TestUtil::NameResolver());
312
313			class BenchmarkTests : public ::testing::TestWithParam<TestCase> {};
314		−	TEST_P(BenchmarkTests, ) { execTestCase(GetParam()); }
315		−	INSTANTIATE_TEST_SUITE_P(, BenchmarkTests, ::testing::ValuesIn(TestUtil::collectTestCases("benchmark", false)),
316			TestUtil::NameResolver());
317
318			class ExampleTests : public ::testing::TestWithParam<TestCase> {};
319		−	TEST_P(ExampleTests, ) { execTestCase(GetParam()); }
320		−	INSTANTIATE_TEST_SUITE_P(, ExampleTests, ::testing::ValuesIn(TestUtil::collectTestCases("examples", false)),
321			TestUtil::NameResolver());
322
323			class BootstrapCompilerTests : public ::testing::TestWithParam<TestCase> {};
324		−	TEST_P(BootstrapCompilerTests, ) { execTestCase(GetParam()); }
325		−	INSTANTIATE_TEST_SUITE_P(, BootstrapCompilerTests, ::testing::ValuesIn(TestUtil::collectTestCases("bootstrap-compiler", false)),
326			TestUtil::NameResolver());
327
328			} // namespace spice::testing
329
330			// GCOV_EXCL_STOP
331