Skip to content

Commit

Permalink
Handle paginated registry metadata responses
Browse files Browse the repository at this point in the history 
In [4.1 List Package
Releases](https://github.com/swiftlang/swift-package-manager/blob/main/Documentation/PackageRegistry/Registry.md#41-list-package-releases)
it is stated that a server may respond with a `Link` header that contains
a pointer to a subsequent page of results. SPM is not checking for this
link in the `Link` header and so if a registry returns paginated results
only the first page of versions is searched when resolving.

Respect the `next` link in the `Link` header by loading the next page of
results and building up a list of versions, continuing until there is no
`next` link present in the `Link` header of the last result.

Issue: swiftlang#8215
  • Loading branch information
@plemarquand
plemarquand committed Jan 15, 2025
1 parent a8495eb commit 2d95b6c
Show file tree
Hide file tree
Showing 3 changed files with 168 additions and 77 deletions.
2 changes: 1 addition & 1 deletion Sources/PackageMetadata/PackageMetadata.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,7 @@ public struct PackageSearchClient {
) { result in
do {
let metadata = try result.get()
let alternateLocations = metadata.alternateLocations ?? []
let alternateLocations = metadata.alternateLocations
return completion(.success(Set(alternateLocations)))
} catch {
return completion(.failure(error))
Expand Down
241 changes: 166 additions & 75 deletions Sources/PackageRegistry/RegistryClient.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,14 +185,16 @@ public final class RegistryClient: Cancellable {
observabilityScope.emit(debug: "registry for \(package): \(registry)")

let underlying = {
self._getPackageMetadata(
registry: registry,
package: registryIdentity,
timeout: timeout,
observabilityScope: observabilityScope,
callbackQueue: callbackQueue,
completion: completion
)
_ = Task {
let result = await self._getPackageMetadata(
registry: registry,
package: registryIdentity,
timeout: timeout,
observabilityScope: observabilityScope,
callbackQueue: callbackQueue
)
completion(result)
}
}

if registry.supportsAvailability {
Expand All @@ -217,64 +219,106 @@ public final class RegistryClient: Cancellable {
package: PackageIdentity.RegistryIdentity,
timeout: DispatchTimeInterval?,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue,
completion: @escaping (Result<PackageMetadata, Error>) -> Void
) {
let completion = self.makeAsync(completion, on: callbackQueue)

callbackQueue: DispatchQueue
) async -> Result<PackageMetadata, Error> {
guard var components = URLComponents(url: registry.url, resolvingAgainstBaseURL: true) else {
return completion(.failure(RegistryError.invalidURL(registry.url)))
return .failure(RegistryError.invalidURL(registry.url))
}
components.appendPathComponents("\(package.scope)", "\(package.name)")
guard let url = components.url else {
return completion(.failure(RegistryError.invalidURL(registry.url)))
return .failure(RegistryError.invalidURL(registry.url))
}

let request = LegacyHTTPClient.Request(
method: .get,
url: url,
headers: [
"Accept": self.acceptHeader(mediaType: .json),
],
options: self.defaultRequestOptions(timeout: timeout, callbackQueue: callbackQueue)
)
// If the responses are paginated then iterate until we've exasuasted all the pages and have a full versions list.
func iterateResponses(url: URL, existingMetadata: PackageMetadata) async -> Result<PackageMetadata, Error> {
let response = await self._getIndividualPackageMetadata(
url: url,
registry: registry,
package: package,
timeout: timeout,
observabilityScope: observabilityScope,
callbackQueue: callbackQueue
)

let start = DispatchTime.now()
observabilityScope.emit(info: "retrieving \(package) metadata from \(request.url)")
self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) { result in
completion(
result.tryMap { response in
observabilityScope
.emit(
debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)"
)
switch response.statusCode {
case 200:
let packageMetadata = try response.parseJSON(
Serialization.PackageMetadata.self,
decoder: self.jsonDecoder
if case .success(let metadata) = response {
let mergedMetadata = PackageMetadata(
registry: registry,
versions: existingMetadata.versions + metadata.versions,
alternateLocations: existingMetadata.alternateLocations.count > 0
? existingMetadata.alternateLocations
: metadata.alternateLocations,
nextPage: metadata.nextPage
)
if let nextPage = mergedMetadata.nextPage?.url {
return await iterateResponses(url: nextPage, existingMetadata: mergedMetadata)
} else {
return .success(
PackageMetadata(
registry: registry,
versions: mergedMetadata.versions.sorted(by: >),
alternateLocations: mergedMetadata.alternateLocations,
nextPage: mergedMetadata.nextPage
)
)
}
}
return response
}

let versions = packageMetadata.releases.filter { $0.value.problem == nil }
.compactMap { Version($0.key) }
.sorted(by: >)
return await iterateResponses(url: url, existingMetadata: PackageMetadata(registry: registry, versions: [], alternateLocations: [], nextPage: nil))
}

let alternateLocations = try response.headers.parseAlternativeLocationLinks()
private func _getIndividualPackageMetadata(
url: URL,
registry: Registry,
package: PackageIdentity.RegistryIdentity,
timeout: DispatchTimeInterval?,
observabilityScope: ObservabilityScope,
callbackQueue: DispatchQueue
) async -> Result<PackageMetadata, Error> {
do {
let start = DispatchTime.now()
observabilityScope.emit(info: "retrieving \(package) metadata from \(url)")
let response = try await self.httpClient.get(
url,
headers: [
"Accept": self.acceptHeader(mediaType: .json),
],
options: self.defaultRequestOptions(timeout: timeout, callbackQueue: callbackQueue),
observabilityScope: observabilityScope
)

return PackageMetadata(
registry: registry,
versions: versions,
alternateLocations: alternateLocations?.map(\.url)
)
case 404:
throw RegistryError.packageNotFound
default:
throw self.unexpectedStatusError(response, expectedStatus: [200, 404])
}
}.mapError {
RegistryError.failedRetrievingReleases(registry: registry, package: package.underlying, error: $0)
observabilityScope
.emit(
debug: "server response for \(url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)"
)

switch response.statusCode {
case 200:
let packageMetadata = try response.parseJSON(
Serialization.PackageMetadata.self,
decoder: self.jsonDecoder
)

let versions = packageMetadata.releases.filter { $0.value.problem == nil }
.compactMap { Version($0.key) }

let alternateLocations = response.headers.parseAlternativeLocationLinks()
let paginationLinks = response.headers.parsePagniationLinks()

return .success(PackageMetadata(
registry: registry,
versions: versions,
alternateLocations: alternateLocations.map(\.url),
nextPage: paginationLinks.first { $0.kind == .next }?.url
))
case 404:
return .failure(RegistryError.failedRetrievingReleases(registry: registry, package: package.underlying, error: RegistryError.packageNotFound))
default:
return .failure(RegistryError.failedRetrievingReleases(registry: registry, package: package.underlying, error: self.unexpectedStatusError(response, expectedStatus: [200, 404])))
}
)
} catch {
return .failure(RegistryError.failedRetrievingReleases(registry: registry, package: package.underlying, error: error))
}
}

Expand Down Expand Up @@ -2071,7 +2115,8 @@ extension RegistryClient {
public struct PackageMetadata {
public let registry: Registry
public let versions: [Version]
public let alternateLocations: [SourceControlURL]?
public let alternateLocations: [SourceControlURL]
public let nextPage: SourceControlURL?
}

public struct PackageVersionMetadata: Sendable {
Expand Down Expand Up @@ -2142,6 +2187,17 @@ extension RegistryClient {
case alternate
}
}

fileprivate struct NextLocationLink {
let url: SourceControlURL
let kind: Kind

enum Kind: String {
// Currently we only care about `next` for pagination, but there are several other values:
// https://github.com/swiftlang/swift-package-manager/blob/0340bb12a56f9696b3966ad82c2aee1594135377/Documentation/PackageRegistry/Registry.md?plain=1#L403-L411
case next
}
}
}

extension RegistryClient {
Expand Down Expand Up @@ -2263,20 +2319,16 @@ extension HTTPClientResponse {
}

extension HTTPClientHeaders {
/*
<https://github.com/mona/LinkedList>; rel="canonical",
<ssh://[email protected]:mona/LinkedList.git>; rel="alternate",
*/
fileprivate func parseAlternativeLocationLinks() throws -> [RegistryClient.AlternativeLocationLink]? {
try self.get("Link").map { header -> [RegistryClient.AlternativeLocationLink] in
fileprivate func parseLink<T>(_ factory: (String) throws -> T?) rethrows -> [T] {
return try self.get("Link").map { header -> [T] in
let linkLines = header.split(separator: ",").map(String.init).map { $0.spm_chuzzle() ?? $0 }
return try linkLines.compactMap { linkLine in
try parseAlternativeLocationLine(linkLine)
try factory(linkLine)
}
}.flatMap { $0 }
}

private func parseAlternativeLocationLine(_ value: String) throws -> RegistryClient.AlternativeLocationLink? {
fileprivate func parseLocationLine<T>(_ value: String, _ factory: (String, String) -> T?) -> T? {
let fields = value.split(separator: ";")
.map(String.init)
.map { $0.spm_chuzzle() ?? $0 }
Expand All @@ -2290,16 +2342,60 @@ extension HTTPClientHeaders {
return nil
}

guard let rel = fields.first(where: { $0.hasPrefix("rel=") }).flatMap({ parseLinkFieldValue($0) }),
let kind = RegistryClient.AlternativeLocationLink.Kind(rawValue: rel)
guard let rel = fields.first(where: { $0.hasPrefix("rel=") }).flatMap({ parseLinkFieldValue($0) })
else {
return nil
}

return RegistryClient.AlternativeLocationLink(
url: SourceControlURL(link),
kind: kind
)
return factory(link, rel)
}
}

extension HTTPClientHeaders {
/*
https://github.com/swiftlang/swift-package-manager/blob/0340bb12a56f9696b3966ad82c2aee1594135377/Documentation/PackageRegistry/Registry.md?plain=1#L395C1-L401C39
<https://github.com/mona/LinkedList>; rel="canonical",
<ssh://[email protected]:mona/LinkedList.git>; rel="alternate",
*/
fileprivate func parseAlternativeLocationLinks() -> [RegistryClient.AlternativeLocationLink] {
self.parseLink(self.parseAlternativeLocationLine(_:))
}

private func parseAlternativeLocationLine(_ value: String) -> RegistryClient.AlternativeLocationLink? {
return parseLocationLine(value) { link, rel in
guard let kind = RegistryClient.AlternativeLocationLink.Kind(rawValue: rel) else {
return nil
}

return RegistryClient.AlternativeLocationLink(
url: SourceControlURL(link),
kind: kind
)
}
}
}

extension HTTPClientHeaders {
/*
https://github.com/swiftlang/swift-package-manager/blob/0340bb12a56f9696b3966ad82c2aee1594135377/Documentation/PackageRegistry/Registry.md?plain=1#L403-L411
<https://github.com/mona/LinkedList?page=2>; rel="next",
<ssh://[email protected]:mona/LinkedList.git?page=40>; rel="last",
*/
fileprivate func parsePagniationLinks() -> [RegistryClient.NextLocationLink] {
self.parseLink(self.parsePaginationLine(_:))
}

private func parsePaginationLine(_ value: String) -> RegistryClient.NextLocationLink? {
return parseLocationLine(value) { link, rel in
guard let kind = RegistryClient.NextLocationLink.Kind(rawValue: rel) else {
return nil
}

return RegistryClient.NextLocationLink(
url: SourceControlURL(link),
kind: kind
)
}
}
}

Expand All @@ -2308,12 +2404,7 @@ extension HTTPClientHeaders {
<http://packages.example.com/mona/LinkedList/1.1.1/Package.swift?swift-version=4>; rel="alternate"; filename="[email protected]"; swift-tools-version="4.0"
*/
fileprivate func parseManifestLinks() throws -> [RegistryClient.ManifestLink] {
try self.get("Link").map { header -> [RegistryClient.ManifestLink] in
let linkLines = header.split(separator: ",").map(String.init).map { $0.spm_chuzzle() ?? $0 }
return try linkLines.compactMap { linkLine in
try parseManifestLinkLine(linkLine)
}
}.flatMap { $0 }
try self.parseLink(self.parseManifestLinkLine(_:))
}

private func parseManifestLinkLine(_ value: String) throws -> RegistryClient.ManifestLink? {
Expand Down
2 changes: 1 addition & 1 deletion Tests/PackageRegistryTests/RegistryClientTests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ final class RegistryClientTests: XCTestCase {
let registryClient = makeRegistryClient(configuration: configuration, httpClient: httpClient)
let metadata = try await registryClient.getPackageMetadata(package: identity)
XCTAssertEqual(metadata.versions, ["1.1.1", "1.0.0"])
XCTAssertEqual(metadata.alternateLocations!, [
XCTAssertEqual(metadata.alternateLocations, [
SourceControlURL("https://github.com/mona/LinkedList"),
SourceControlURL("ssh://[email protected]:mona/LinkedList.git"),
SourceControlURL("[email protected]:mona/LinkedList.git"),
Expand Down

0 comments on commit 2d95b6c

Please sign in to comment.