generator/linux: Use swiftResourcesPath to find framework headers

Building with a Swift 6.0 SDK fails because the new `_FoundationCShims` framework headers can't be found: <unknown>:0: error: missing required module '_FoundationCShims' The necessary files are present in `$PLATFORM.sdk/usr/lib/swift`, but are not part of the list which is copied to `$PLATFORM.sdk/usr/include`. The Static Linux SDK and WASM SDK generator don't copy these files into /usr/include; instead they set the `swiftResourcesPath` and `swiftStaticResourcesPath` fields in `swift-sdk.json`, which causes the build system to look in those paths for framework headers. We can do the same for Linux SDKs, however we must stop copying the files to `/usr/include`, otherwise the build will fail because the framework header definitions are duplicated: error: redefinition of module 'DispatchIntrospection' 6 | } 7 | 8 | module DispatchIntrospection [system] [extern_c] { | `- error: redefinition of module 'DispatchIntrospection' 9 | header "introspection.h" 10 | export * This change fixes issue swiftlang#152. Compared to swiftlang#153, the end-to-end tests now pass for all supported Swift versions, distributions and architectures. | SDK | Hello World | Foundation | | ----------------------------------------- | ----------- | ---------- | | ubuntu_aarch64_5.9.2-RELEASE | ok | ok | | ubuntu_aarch64_5.9.2-RELEASE_with-docker | ok | ok | | ubuntu_aarch64_5.10.1-RELEASE | ok | ok | | ubuntu_aarch64_5.10.1-RELEASE_with-docker | ok | ok | | ubuntu_aarch64_6.0.2-RELEASE | ok | ok | | ubuntu_aarch64_6.0.2-RELEASE_with-docker | ok | ok | | | | | | ubuntu_x86_64_5.9.2-RELEASE | ok | ok | | ubuntu_x86_64_5.9.2-RELEASE_with-docker | ok | ok | | ubuntu_x86_64_5.10.1-RELEASE | ok | ok | | ubuntu_x86_64_5.10.1-RELEASE_with-docker | ok | ok | | ubuntu_x86_64_6.0.2-RELEASE | ok | ok | | ubuntu_x86_64_6.0.2-RELEASE_with-docker | ok | ok | | | | | | rhel_aarch64_5.9.2-RELEASE_with-docker | ok | ok | | rhel_aarch64_5.10.1-RELEASE_with-docker | ok | ok | | rhel_aarch64_6.0.2-RELEASE_with-docker | ok | ok | | | | | | rhel_x86_64_5.9.2-RELEASE_with-docker | ok | ok | | rhel_x86_64_5.10.1-RELEASE_with-docker | ok | ok | | rhel_x86_64_6.0.2-RELEASE_with-docker | ok | ok | FAIL1: cannot find /lib/ld-linux-aarch64.so.1 (swiftlang#147) FAIL2: missing required module '_FoundationCShims' (swiftlang#152) Fixes: swiftlang#152 Depends on: swiftlang#153, swiftlang#154
euanh · Nov 27, 2024 · 77671c9 · 77671c9
1 parent cc30535
commit 77671c9
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 68 deletions.
diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift
@@ -97,7 +97,7 @@ extension SwiftSDKGenerator {
         try await generator.removeRecursively(at: sdkUsrLibPath.appending("python3.10"))
 
         try await generator.removeRecursively(at: sdkUsrLibPath.appending("ssl"))
-        try await generator.copyTargetSwift(from: sdkUsrLibPath, sdkDirPath: sdkDirPath)
+        try await generator.copyTargetSwift(from: sdkUsrPath, sdkDirPath: sdkDirPath)
       }
     }
   }
@@ -106,12 +106,10 @@ extension SwiftSDKGenerator {
     logGenerationStep("Copying Swift core libraries for the target triple into Swift SDK bundle...")
 
     for (pathWithinPackage, pathWithinSwiftSDK) in [
-      ("swift/linux", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift")),
-      ("swift_static/linux", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static")),
-      ("swift_static/shims", pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static")),
-      ("swift/dispatch", sdkDirPath.appending("usr/include")),
-      ("swift/os", sdkDirPath.appending("usr/include")),
-      ("swift/CoreFoundation", sdkDirPath.appending("usr/include")),
+      ("lib/swift", sdkDirPath.appending("usr/lib")),
+      ("lib/swift_static", sdkDirPath.appending("usr/lib")),
+      ("lib/clang", sdkDirPath.appending("usr/lib")),
+      ("include", sdkDirPath.appending("usr")),
     ] {
       try await rsync(from: distributionPath.appending(pathWithinPackage), to: pathWithinSwiftSDK)
     }

diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift
@@ -48,44 +48,6 @@ extension SwiftSDKGenerator {
     }
   }
 
-  func fixGlibcModuleMap(at path: FilePath, hostTriple: Triple) throws {
-    logGenerationStep("Fixing absolute paths in `glibc.modulemap`...")
-
-    guard doesFileExist(at: path) else {
-      throw GeneratorError.fileDoesNotExist(path)
-    }
-
-    let privateIncludesPath = path.removingLastComponent().appending("private_includes")
-    try removeRecursively(at: privateIncludesPath)
-    try createDirectoryIfNeeded(at: privateIncludesPath)
-
-    let regex = Regex {
-      #/\n( *header )"\/+usr\/include\//#
-      Capture {
-        Optionally {
-          hostTriple.arch!.linuxConventionName
-          "-linux-gnu"
-        }
-      }
-      #/([^\"]+)\"/#
-    }
-
-    var moduleMap = try String(data: readFile(at: path), encoding: .utf8)!
-    try moduleMap.replace(regex) {
-      let (_, headerKeyword, _, headerPath) = $0.output
-
-      let newHeaderRelativePath = headerPath.replacing("/", with: "_")
-      try writeFile(
-        at: privateIncludesPath.appending(String(newHeaderRelativePath)),
-        Data("#include <linux/uuid.h>\n".utf8)
-      )
-
-      return #"\#n\#(headerKeyword) "private_includes/\#(newHeaderRelativePath)""#
-    }
-
-    try writeFile(at: path, Data(moduleMap.utf8))
-  }
-
   func symlinkClangHeaders() throws {
     try self.createSymlink(
       at: self.pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static/clang"),

diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift
@@ -86,7 +86,7 @@ extension SwiftSDKGenerator {
     try await inTemporaryDirectory { fs, tmpDir in
       try await fs.unpack(file: targetSwiftPackagePath, into: tmpDir)
       try await fs.copyTargetSwift(
-        from: tmpDir.appending(relativePathToRoot).appending("usr/lib"), sdkDirPath: sdkDirPath
+        from: tmpDir.appending(relativePathToRoot).appending("usr"), sdkDirPath: sdkDirPath
       )
     }
   }

diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift
@@ -122,6 +122,19 @@ public struct LinuxRecipe: SwiftSDKRecipe {
     toolset.librarian = Toolset.ToolProperties(path: "llvm-ar")
   }
 
+  public func applyPlatformOptions(
+    metadata: inout SwiftSDKMetadataV4.TripleProperties,
+    paths: PathsConfiguration,
+    targetTriple: Triple
+  ) {
+    var relativeSDKDir = self.sdkDirPath(paths: paths)
+    guard relativeSDKDir.removePrefix(paths.swiftSDKRootPath) else {
+      fatalError("The SDK directory path must be a subdirectory of the Swift SDK root path.")
+    }
+    metadata.swiftResourcesPath = relativeSDKDir.appending("usr/lib/swift").string
+    metadata.swiftStaticResourcesPath = relativeSDKDir.appending("usr/lib/swift_static").string
+  }
+
   public var defaultArtifactID: String {
     """
     \(self.versionsConfiguration.swiftVersion)_\(self.linuxDistribution.name.rawValue)_\(
@@ -211,7 +224,7 @@ public struct LinuxRecipe: SwiftSDKRecipe {
       )
     case let .localPackage(filePath):
       try await generator.copyTargetSwift(
-        from: filePath.appending("usr/lib"), sdkDirPath: sdkDirPath
+        from: filePath.appending("usr"), sdkDirPath: sdkDirPath
       )
     case .remoteTarball:
       try await generator.unpackTargetSwiftPackage(
@@ -225,13 +238,6 @@ public struct LinuxRecipe: SwiftSDKRecipe {
 
     try await generator.fixAbsoluteSymlinks(sdkDirPath: sdkDirPath)
 
-    let targetCPU = generator.targetTriple.arch!
-    try await generator.fixGlibcModuleMap(
-      at: generator.pathsConfiguration.toolchainDirPath
-        .appending("/usr/lib/swift/linux/\(targetCPU.linuxConventionName)/glibc.modulemap"),
-      hostTriple: self.mainHostTriple
-    )
-
     if self.versionsConfiguration.swiftVersion.hasPrefix("5.9") ||
         self.versionsConfiguration.swiftVersion .hasPrefix("5.10") {
       try await generator.symlinkClangHeaders()

diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift
@@ -187,14 +187,6 @@ func skipSlow() throws {
   )
 }
 
-// Skip known failing tests unless an environment variable is set
-func skipBroken(_ message: String) throws {
-  try XCTSkipUnless(
-    ProcessInfo.processInfo.environment.keys.contains("SWIFT_SDK_GENERATOR_RUN_BROKEN_TESTS"),
-    "Skipping broken test because SWIFT_SDK_GENERATOR_RUN_BROKEN_TESTS is not set: \(message)"
-  )
-}
-
 func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempDir: URL) async throws {
   let testPackageURL = tempDir.appendingPathComponent("swift-sdk-generator-test")
   let testPackageDir = FilePath(testPackageURL.path)
@@ -318,25 +310,21 @@ final class Swift60_UbuntuEndToEndTests: XCTestCase {
   )
 
   func testAarch64Direct() async throws {
-    try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152")
     try skipSlow()
     try await buildTestcases(config: config.withArchitecture("aarch64"))
   }
 
   func testX86_64Direct() async throws {
-    try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152")
     try skipSlow()
     try await buildTestcases(config: config.withArchitecture("x86_64"))
   }
 
   func testAarch64FromContainer() async throws {
-    try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152")
     try skipSlow()
     try await buildTestcases(config: config.withArchitecture("aarch64").withDocker())
   }
 
   func testX86_64FromContainer() async throws {
-    try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152")
     try skipSlow()
     try await buildTestcases(config: config.withArchitecture("x86_64").withDocker())
   }
@@ -389,13 +377,11 @@ final class Swift60_RHELEndToEndTests: XCTestCase {
   )
 
   func testAarch64FromContainer() async throws {
-    try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152")
     try skipSlow()
     try await buildTestcases(config: config.withArchitecture("aarch64").withDocker())
   }
 
   func testX86_64FromContainer() async throws {
-    try skipBroken("https://github.com/swiftlang/swift-sdk-generator/issues/152")
     try skipSlow()
     try await buildTestcases(config: config.withArchitecture("x86_64").withDocker())
   }