From b7cf646988d4d1bcc685a677c8d4770737c3cee9 Mon Sep 17 00:00:00 2001 From: Euan Harris Date: Wed, 20 Nov 2024 08:39:24 +0000 Subject: [PATCH] tests: Refactor and expand end-to-end test coverage, disabled by default This commit extends the end-to-end tests to cover all currently-supported combinations of Swift version, Linux distribution and CPU architecture. For Ubuntu, there are also tests of SDKs built directly from packages and from container images. These tests run slowly (about 30 minutes on my machine), so they are disabled unless the SWIFT_SDK_GENERATOR_RUN_SLOW_TESTS environment variable is set. There is further scope for speeding up the generation of SDKs from container images, which might allow more tests to be included by default. Quite a lot of time, on macOS, is also spent waiting for the toolchain embedded in the SDK to be verified - this is more difficult to mitigate. Splitting the tests up makes it easier to see problems which affect particular configurations. Each integration test case tries to build two different programs; ideally these would be reported as separate test cases, but for now some manual re-testing is needed to separate the causes of test failures. Currently there are two problems on main which cause tests to fail: * Issue #147 causes ld-linux-aarch64.so.1 not to be copied into SDKs built from containers. Attempted builds using these SDKs fails at the final link stage. * Issue #152 causes new shims not to be copied into all Swift 6.0 SDKs. Attempted builds with these SDKs fail during compilation stage, if Foundation is used. In some cases issue #152 masks issue #147, because it occurs earlier in the build process. For instance, using ubuntu_aarch64_6.0.2-RELEASE_with-docker to build the 'Foundation' test case currently fails because of #152, but if that were to be fixed it would then still fail because of #147. | SDK | Hello World | Foundation | | ----------------------------------------- | ----------- | ---------- | | ubuntu_aarch64_5.9.2-RELEASE | ok | ok | | ubuntu_aarch64_5.9.2-RELEASE_with-docker | FAIL1 | FAIL1 | | ubuntu_aarch64_5.10.1-RELEASE | ok | ok | | ubuntu_aarch64_5.10.1-RELEASE_with-docker | FAIL1 | FAIL1 | | ubuntu_aarch64_6.0.2-RELEASE | ok | FAIL2 | | ubuntu_aarch64_6.0.2-RELEASE_with-docker | FAIL1 | FAIL2 | | | | | | ubuntu_x86_64_5.9.2-RELEASE | ok | ok | | ubuntu_x86_64_5.9.2-RELEASE_with-docker | ok | ok | | ubuntu_x86_64_5.10.1-RELEASE | ok | ok | | ubuntu_x86_64_5.10.1-RELEASE_with-docker | ok | ok | | ubuntu_x86_64_6.0.2-RELEASE | ok | FAIL2 | | ubuntu_x86_64_6.0.2-RELEASE_with-docker | ok | FAIL2 | | | | | | rhel_aarch64_5.9.2-RELEASE_with-docker | FAIL1 | FAIL1 | | rhel_aarch64_5.10.1-RELEASE_with-docker | FAIL1 | FAIL1 | | rhel_aarch64_6.0.2-RELEASE_with-docker | FAIL1 | FAIL2 | | | | | | rhel_x86_64_5.9.2-RELEASE_with-docker | ok | ok | | rhel_x86_64_5.10.1-RELEASE_with-docker | ok | ok | | rhel_x86_64_6.0.2-RELEASE_with-docker | ok | FAIL2 | FAIL1: cannot find /lib/ld-linux-aarch64.so.1 (Issue #147) FAIL2: missing required module '_FoundationCShims' (Issue #152) --- .../EndToEndTests.swift | 400 ++++++++++++++---- 1 file changed, 308 insertions(+), 92 deletions(-) diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 8f67136..7365969 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -49,66 +49,80 @@ extension FileManager { } } -final class EndToEndTests: XCTestCase { - private let testcases = [ - #""" - // Default program generated by swift package init - print("Hello, world!") - """#, - #""" - // Check that libc_nonshared.a is linked properly - import Foundation - - func fin() -> Void { - print("exiting") - } - - atexit(fin) - """#, - ] - - private let logger = Logger(label: "swift-sdk-generator") - - // Building an SDK requires running the sdk-generator with `swift run swift-sdk-generator`. - // This takes a lock on `.build`, but if the tests are being run by `swift test` the outer Swift Package Manager - // instance will already hold this lock, causing the test to deadlock. We can work around this by giving - // the `swift run swift-sdk-generator` instance its own scratch directory. - func buildSDK(inDirectory packageDirectory: FilePath, scratchPath: String, withArguments runArguments: String) async throws -> String { - let generatorOutput = try await Shell.readStdout( - "cd \(packageDirectory) && swift run --scratch-path \"\(scratchPath)\" swift-sdk-generator make-linux-sdk \(runArguments)" - ) - - let installCommand = try XCTUnwrap(generatorOutput.split(separator: "\n").first { - $0.contains("swift experimental-sdk install") - }) +// Building an SDK requires running the sdk-generator with `swift run swift-sdk-generator`. +// This takes a lock on `.build`, but if the tests are being run by `swift test` the outer Swift Package Manager +// instance will already hold this lock, causing the test to deadlock. We can work around this by giving +// the `swift run swift-sdk-generator` instance its own scratch directory. +func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: String) async throws -> String { + var logger = logger + logger[metadataKey: "runArguments"] = "\"\(runArguments)\"" + logger[metadataKey: "scratchPath"] = "\(scratchPath)" + + logger.info("Building SDK") + + var packageDirectory = FilePath(#filePath) + packageDirectory.removeLastComponent() + packageDirectory.removeLastComponent() + + let generatorOutput = try await Shell.readStdout( + "cd \(packageDirectory) && swift run --scratch-path \"\(scratchPath)\" swift-sdk-generator make-linux-sdk \(runArguments)" + ) + logger.info("Finished building SDK") + + let installCommand = try XCTUnwrap(generatorOutput.split(separator: "\n").first { + $0.contains("swift experimental-sdk install") + }) + + let bundleName = try XCTUnwrap( + FilePath(String(XCTUnwrap(installCommand.split(separator: " ").last))).components.last + ).stem + logger[metadataKey: "bundleName"] = "\(bundleName)" + + logger.info("Checking installed SDKs") + let installedSDKs = try await Shell.readStdout("swift experimental-sdk list").components(separatedBy: "\n") + + // Make sure this bundle hasn't been installed already. + if installedSDKs.contains(bundleName) { + logger.info("Removing existing SDK") + try await Shell.run("swift experimental-sdk remove \(bundleName)") + } - let bundleName = try XCTUnwrap( - FilePath(String(XCTUnwrap(installCommand.split(separator: " ").last))).components.last - ).stem + logger.info("Installing new SDK") + let installOutput = try await Shell.readStdout(String(installCommand)) + XCTAssertTrue(installOutput.contains("successfully installed")) - let installedSDKs = try await Shell.readStdout("swift experimental-sdk list").components(separatedBy: "\n") + return bundleName +} - // Make sure this bundle hasn't been installed already. - if installedSDKs.contains(bundleName) { - try await Shell.run("swift experimental-sdk remove \(bundleName)") +private let testcases = [ + #""" + // Default program generated by swift package init + print("Hello, world!") + """#, + #""" + // Check that libc_nonshared.a is linked properly + import Foundation + + func fin() -> Void { + print("exiting") } - let installOutput = try await Shell.readStdout(String(installCommand)) - XCTAssertTrue(installOutput.contains("successfully installed")) + atexit(fin) + """#, +] - return bundleName - } +final class RepeatedBuildTests: XCTestCase { + private let logger = Logger(label: "swift-sdk-generator") - func testPackageInitExecutable() async throws { + func testRepeatedSDKBuilds() async throws { if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { throw XCTSkip("EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145") } - var packageDirectory = FilePath(#filePath) - packageDirectory.removeLastComponent() - packageDirectory.removeLastComponent() + var logger = logger + logger[metadataKey: "testcase"] = "testRepeatedSDKBuilds" - // Do multiple runs with different sets of arguments. + // Test that an existing SDK can be rebuilt without cleaning up. // Test with no arguments by default: var possibleArguments = [""] do { @@ -125,66 +139,268 @@ final class EndToEndTests: XCTestCase { continue } - let bundleName = try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in - try await buildSDK(inDirectory: packageDirectory, scratchPath: tempDir.path, withArguments: runArguments) + try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in + let _ = try await buildSDK(logger, scratchPath: tempDir.path, withArguments: runArguments) + let _ = try await buildSDK(logger, scratchPath: tempDir.path, withArguments: runArguments) } + } + } +} - for testcase in self.testcases { - try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in - let testPackageURL = tempDir.appendingPathComponent("swift-sdk-generator-test") - let testPackageDir = FilePath(testPackageURL.path) - try FileManager.default.createDirectory(atPath: testPackageDir.string, withIntermediateDirectories: true) +// SDKConfiguration represents an SDK build configuration and can construct the corresponding SDK generator arguments +struct SDKConfiguration { + var swiftVersion: String + var linuxDistributionName: String + var architecture: String + var withDocker: Bool - try await Shell.run("swift package --package-path \(testPackageDir) init --type executable") - let main_swift = testPackageURL.appendingPathComponent("Sources/main.swift") - try testcase.write(to: main_swift, atomically: true, encoding: .utf8) + var bundleName: String { "\(linuxDistributionName)_\(architecture)_\(swiftVersion)-RELEASE\(withDocker ? "_with-docker" : "")" } - var buildOutput = try await Shell.readStdout( - "swift build --package-path \(testPackageDir) --experimental-swift-sdk \(bundleName)" - ) - XCTAssertTrue(buildOutput.contains("Build complete!")) + func withDocker(_ enabled: Bool = true) -> SDKConfiguration { + var res = self + res.withDocker = enabled + return res + } - try await Shell.run("rm -rf \(testPackageDir.appending(".build"))") + func withArchitecture(_ arch: String) -> SDKConfiguration { + var res = self + res.architecture = arch + return res + } - buildOutput = try await Shell.readStdout( - "swift build --package-path \(testPackageDir) --experimental-swift-sdk \(bundleName) --static-swift-stdlib" - ) - XCTAssertTrue(buildOutput.contains("Build complete!")) - } - } - } + var sdkGeneratorArguments: String { + return [ + "--sdk-name \(bundleName)", + withDocker ? "--with-docker" : nil, + "--swift-version \(swiftVersion)-RELEASE", + "--target \(architecture)-unknown-linux-gnu", + "--linux-distribution-name \(linuxDistributionName)" + ].compactMap{ $0 }.joined(separator: " ") } +} - func testRepeatedSDKBuilds() async throws { - if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { - throw XCTSkip("EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145") - } +// Skip slow tests unless an environment variable is set +func skipSlow() throws { + try XCTSkipUnless( + ProcessInfo.processInfo.environment.keys.contains("SWIFT_SDK_GENERATOR_RUN_SLOW_TESTS"), + "Skipping slow test because SWIFT_SDK_GENERATOR_RUN_SLOW_TESTS is not set" + ) +} - var packageDirectory = FilePath(#filePath) - packageDirectory.removeLastComponent() - packageDirectory.removeLastComponent() +// Skip known failing tests unless an environment variable is set +func skipBroken(_ message: String) throws { + try XCTSkipUnless( + ProcessInfo.processInfo.environment.keys.contains("SWIFT_SDK_GENERATOR_RUN_BROKEN_TESTS"), + "Skipping broken test because SWIFT_SDK_GENERATOR_RUN_BROKEN_TESTS is not set: \(message)" + ) +} - // Test that an existing SDK can be rebuilt without cleaning up. - // Test with no arguments by default: - var possibleArguments = [""] +func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempDir: URL) async throws { + let testPackageURL = tempDir.appendingPathComponent("swift-sdk-generator-test") + let testPackageDir = FilePath(testPackageURL.path) + try FileManager.default.createDirectory(atPath: testPackageDir.string, withIntermediateDirectories: true) + + logger.info("Creating test project") + try await Shell.run("swift package --package-path \(testPackageDir) init --type executable") + let main_swift = testPackageURL.appendingPathComponent("Sources/main.swift") + try testcase.write(to: main_swift, atomically: true, encoding: .utf8) + + logger.info("Building test project") + var buildOutput = try await Shell.readStdout( + "swift build --package-path \(testPackageDir) --experimental-swift-sdk \(bundleName)" + ) + XCTAssertTrue(buildOutput.contains("Build complete!")) + logger.info("Test project built successfully") + + try await Shell.run("rm -rf \(testPackageDir.appending(".build"))") + + logger.info("Building test project with static-swift-stdlib") + buildOutput = try await Shell.readStdout( + "swift build --package-path \(testPackageDir) --experimental-swift-sdk \(bundleName) --static-swift-stdlib" + ) + XCTAssertTrue(buildOutput.contains("Build complete!")) + logger.info("Test project built successfully") +} + +func buildTestcases(config: SDKConfiguration) async throws { + var logger = Logger(label: "EndToEndTests") + logger[metadataKey: "testcase"] = "testPackageInitExecutable" + + if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { + throw XCTSkip("EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145") + } + + if config.withDocker { do { try await Shell.run("docker ps") - possibleArguments.append("--with-docker --linux-distribution-name rhel --linux-distribution-version ubi9") } catch { - self.logger.warning("Docker CLI does not seem to be working, skipping tests that involve Docker.") + throw XCTSkip("Container runtime is not available - skipping tests which require it") } + } - for runArguments in possibleArguments { - if runArguments.contains("rhel") { - // Temporarily skip the RHEL-based SDK. XCTSkip() is not suitable as it would skipping the entire test case - logger.warning("RHEL-based SDKs currently do not work with Swift 6.0: https://github.com/swiftlang/swift-sdk-generator/issues/138") - continue - } + let bundleName = try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in + try await buildSDK(logger, scratchPath: tempDir.path, withArguments: config.sdkGeneratorArguments) + } - try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in - let _ = try await buildSDK(inDirectory: packageDirectory, scratchPath: tempDir.path, withArguments: runArguments) - let _ = try await buildSDK(inDirectory: packageDirectory, scratchPath: tempDir.path, withArguments: runArguments) - } + logger.info("Built SDK") + + for testcase in testcases { + try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in + try await buildTestcase(logger, testcase: testcase, bundleName: bundleName, tempDir: tempDir) } } } + +final class Swift59_UbuntuEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.9.2", + linuxDistributionName: "ubuntu", + architecture: "aarch64", + withDocker: false + ) + + func testAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAarch64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/147") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } +} + +final class Swift510_UbuntuEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.10.1", + linuxDistributionName: "ubuntu", + architecture: "aarch64", + withDocker: false + ) + + func testAarch64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testX86_64Direct() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAarch64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/147") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } +} + +final class Swift60_UbuntuEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.0.2", + linuxDistributionName: "ubuntu", + architecture: "aarch64", + withDocker: false + ) + + func testAarch64Direct() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64")) + } + + func testX86_64Direct() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64")) + } + + func testAarch64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } +} + +final class Swift59_RHELEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.9.2", + linuxDistributionName: "rhel", + architecture: "aarch64", + withDocker: true // RHEL-based SDKs can only be built from containers + ) + + func testAarch64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/147") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } +} + +final class Swift510_RHELEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "5.10.1", + linuxDistributionName: "rhel", + architecture: "aarch64", + withDocker: true // RHEL-based SDKs can only be built from containers + ) + + func testAarch64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/147") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } +} + +final class Swift60_RHELEndToEndTests: XCTestCase { + let config = SDKConfiguration( + swiftVersion: "6.0.2", + linuxDistributionName: "rhel", + architecture: "aarch64", + withDocker: true // RHEL-based SDKs can only be built from containers + ) + + func testAarch64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/147") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) + } + + func testX86_64FromContainer() async throws { + try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") + try skipSlow() + try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) + } +}