From 77671c9168ec8b4b9882c9a54c24167df8f6f2be Mon Sep 17 00:00:00 2001 From: Euan Harris Date: Mon, 25 Nov 2024 11:10:05 +0000 Subject: [PATCH] generator/linux: Use swiftResourcesPath to find framework headers Building with a Swift 6.0 SDK fails because the new `_FoundationCShims` framework headers can't be found: :0: error: missing required module '_FoundationCShims' The necessary files are present in `$PLATFORM.sdk/usr/lib/swift`, but are not part of the list which is copied to `$PLATFORM.sdk/usr/include`. The Static Linux SDK and WASM SDK generator don't copy these files into /usr/include; instead they set the `swiftResourcesPath` and `swiftStaticResourcesPath` fields in `swift-sdk.json`, which causes the build system to look in those paths for framework headers. We can do the same for Linux SDKs, however we must stop copying the files to `/usr/include`, otherwise the build will fail because the framework header definitions are duplicated: error: redefinition of module 'DispatchIntrospection' 6 | } 7 | 8 | module DispatchIntrospection [system] [extern_c] { | `- error: redefinition of module 'DispatchIntrospection' 9 | header "introspection.h" 10 | export * This change fixes issue #152. Compared to #153, the end-to-end tests now pass for all supported Swift versions, distributions and architectures. | SDK | Hello World | Foundation | | ----------------------------------------- | ----------- | ---------- | | ubuntu_aarch64_5.9.2-RELEASE | ok | ok | | ubuntu_aarch64_5.9.2-RELEASE_with-docker | ok | ok | | ubuntu_aarch64_5.10.1-RELEASE | ok | ok | | ubuntu_aarch64_5.10.1-RELEASE_with-docker | ok | ok | | ubuntu_aarch64_6.0.2-RELEASE | ok | ok | | ubuntu_aarch64_6.0.2-RELEASE_with-docker | ok | ok | | | | | | ubuntu_x86_64_5.9.2-RELEASE | ok | ok | | ubuntu_x86_64_5.9.2-RELEASE_with-docker | ok | ok | | ubuntu_x86_64_5.10.1-RELEASE | ok | ok | | ubuntu_x86_64_5.10.1-RELEASE_with-docker | ok | ok | | ubuntu_x86_64_6.0.2-RELEASE | ok | ok | | ubuntu_x86_64_6.0.2-RELEASE_with-docker | ok | ok | | | | | | rhel_aarch64_5.9.2-RELEASE_with-docker | ok | ok | | rhel_aarch64_5.10.1-RELEASE_with-docker | ok | ok | | rhel_aarch64_6.0.2-RELEASE_with-docker | ok | ok | | | | | | rhel_x86_64_5.9.2-RELEASE_with-docker | ok | ok | | rhel_x86_64_5.10.1-RELEASE_with-docker | ok | ok | | rhel_x86_64_6.0.2-RELEASE_with-docker | ok | ok | FAIL1: cannot find /lib/ld-linux-aarch64.so.1 (#147) FAIL2: missing required module '_FoundationCShims' (#152) Fixes: #152 Depends on: #153, #154 --- .../Generator/SwiftSDKGenerator+Copy.swift | 12 +++--- .../Generator/SwiftSDKGenerator+Fixup.swift | 38 ------------------- .../Generator/SwiftSDKGenerator+Unpack.swift | 2 +- .../SwiftSDKRecipes/LinuxRecipe.swift | 22 +++++++---- .../EndToEndTests.swift | 14 ------- 5 files changed, 20 insertions(+), 68 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift index 934b2a5..29d4f4a 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift @@ -97,7 +97,7 @@ extension SwiftSDKGenerator { try await generator.removeRecursively(at: sdkUsrLibPath.appending("python3.10")) try await generator.removeRecursively(at: sdkUsrLibPath.appending("ssl")) - try await generator.copyTargetSwift(from: sdkUsrLibPath, sdkDirPath: sdkDirPath) + try await generator.copyTargetSwift(from: sdkUsrPath, sdkDirPath: sdkDirPath) } } } @@ -106,12 +106,10 @@ extension SwiftSDKGenerator { logGenerationStep("Copying Swift core libraries for the target triple into Swift SDK bundle...") for (pathWithinPackage, pathWithinSwiftSDK) in [ - ("swift/linux", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift")), - ("swift_static/linux", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static")), - ("swift_static/shims", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static")), - ("swift/dispatch", sdkDirPath.appending("usr/include")), - ("swift/os", sdkDirPath.appending("usr/include")), - ("swift/CoreFoundation", sdkDirPath.appending("usr/include")), + ("lib/swift", sdkDirPath.appending("usr/lib")), + ("lib/swift_static", sdkDirPath.appending("usr/lib")), + ("lib/clang", sdkDirPath.appending("usr/lib")), + ("include", sdkDirPath.appending("usr")), ] { try await rsync(from: distributionPath.appending(pathWithinPackage), to: pathWithinSwiftSDK) } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift index fe01383..fd195b1 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift @@ -48,44 +48,6 @@ extension SwiftSDKGenerator { } } - func fixGlibcModuleMap(at path: FilePath, hostTriple: Triple) throws { - logGenerationStep("Fixing absolute paths in `glibc.modulemap`...") - - guard doesFileExist(at: path) else { - throw GeneratorError.fileDoesNotExist(path) - } - - let privateIncludesPath = path.removingLastComponent().appending("private_includes") - try removeRecursively(at: privateIncludesPath) - try createDirectoryIfNeeded(at: privateIncludesPath) - - let regex = Regex { - #/\n( *header )"\/+usr\/include\//# - Capture { - Optionally { - hostTriple.arch!.linuxConventionName - "-linux-gnu" - } - } - #/([^\"]+)\"/# - } - - var moduleMap = try String(data: readFile(at: path), encoding: .utf8)! - try moduleMap.replace(regex) { - let (_, headerKeyword, _, headerPath) = $0.output - - let newHeaderRelativePath = headerPath.replacing("/", with: "_") - try writeFile( - at: privateIncludesPath.appending(String(newHeaderRelativePath)), - Data("#include \n".utf8) - ) - - return #"\#n\#(headerKeyword) "private_includes/\#(newHeaderRelativePath)""# - } - - try writeFile(at: path, Data(moduleMap.utf8)) - } - func symlinkClangHeaders() throws { try self.createSymlink( at: self.pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static/clang"), diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift index 874ed2a..47d4f30 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift @@ -86,7 +86,7 @@ extension SwiftSDKGenerator { try await inTemporaryDirectory { fs, tmpDir in try await fs.unpack(file: targetSwiftPackagePath, into: tmpDir) try await fs.copyTargetSwift( - from: tmpDir.appending(relativePathToRoot).appending("usr/lib"), sdkDirPath: sdkDirPath + from: tmpDir.appending(relativePathToRoot).appending("usr"), sdkDirPath: sdkDirPath ) } } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index cb9c7e9..35ffb08 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -122,6 +122,19 @@ public struct LinuxRecipe: SwiftSDKRecipe { toolset.librarian = Toolset.ToolProperties(path: "llvm-ar") } + public func applyPlatformOptions( + metadata: inout SwiftSDKMetadataV4.TripleProperties, + paths: PathsConfiguration, + targetTriple: Triple + ) { + var relativeSDKDir = self.sdkDirPath(paths: paths) + guard relativeSDKDir.removePrefix(paths.swiftSDKRootPath) else { + fatalError("The SDK directory path must be a subdirectory of the Swift SDK root path.") + } + metadata.swiftResourcesPath = relativeSDKDir.appending("usr/lib/swift").string + metadata.swiftStaticResourcesPath = relativeSDKDir.appending("usr/lib/swift_static").string + } + public var defaultArtifactID: String { """ \(self.versionsConfiguration.swiftVersion)_\(self.linuxDistribution.name.rawValue)_\( @@ -211,7 +224,7 @@ public struct LinuxRecipe: SwiftSDKRecipe { ) case let .localPackage(filePath): try await generator.copyTargetSwift( - from: filePath.appending("usr/lib"), sdkDirPath: sdkDirPath + from: filePath.appending("usr"), sdkDirPath: sdkDirPath ) case .remoteTarball: try await generator.unpackTargetSwiftPackage( @@ -225,13 +238,6 @@ public struct LinuxRecipe: SwiftSDKRecipe { try await generator.fixAbsoluteSymlinks(sdkDirPath: sdkDirPath) - let targetCPU = generator.targetTriple.arch! - try await generator.fixGlibcModuleMap( - at: generator.pathsConfiguration.toolchainDirPath - .appending("/usr/lib/swift/linux/\(targetCPU.linuxConventionName)/glibc.modulemap"), - hostTriple: self.mainHostTriple - ) - if self.versionsConfiguration.swiftVersion.hasPrefix("5.9") || self.versionsConfiguration.swiftVersion .hasPrefix("5.10") { try await generator.symlinkClangHeaders() diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 2aef1c3..49951eb 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -187,14 +187,6 @@ func skipSlow() throws { ) } -// Skip known failing tests unless an environment variable is set -func skipBroken(_ message: String) throws { - try XCTSkipUnless( - ProcessInfo.processInfo.environment.keys.contains("SWIFT_SDK_GENERATOR_RUN_BROKEN_TESTS"), - "Skipping broken test because SWIFT_SDK_GENERATOR_RUN_BROKEN_TESTS is not set: \(message)" - ) -} - func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempDir: URL) async throws { let testPackageURL = tempDir.appendingPathComponent("swift-sdk-generator-test") let testPackageDir = FilePath(testPackageURL.path) @@ -318,25 +310,21 @@ final class Swift60_UbuntuEndToEndTests: XCTestCase { ) func testAarch64Direct() async throws { - try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") try skipSlow() try await buildTestcases(config: config.withArchitecture("aarch64")) } func testX86_64Direct() async throws { - try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") try skipSlow() try await buildTestcases(config: config.withArchitecture("x86_64")) } func testAarch64FromContainer() async throws { - try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") try skipSlow() try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) } func testX86_64FromContainer() async throws { - try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") try skipSlow() try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) } @@ -389,13 +377,11 @@ final class Swift60_RHELEndToEndTests: XCTestCase { ) func testAarch64FromContainer() async throws { - try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") try skipSlow() try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) } func testX86_64FromContainer() async throws { - try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152") try skipSlow() try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) }