Add GitLab as metadata-provider and make it easier to support other ones

Sources/PackageCollectionGenerator/Models/PackageCollectionGeneratorInput.swift

@@ -33,18 +33,23 @@ public struct PackageCollectionGeneratorInput: Equatable, Codable {
     /// The author of this package collection.
     public let author: PackageCollectionModel.V1.Collection.Author?
 
+    /// A mapping of the hosts to the corresponding metadata-provider (GitHub, GitLab, ...).
+    public let metadataProviders: [String: String]?
+
     public init(
         name: String,
         overview: String? = nil,
         keywords: [String]? = nil,
         packages: [Package],
-        author: PackageCollectionModel.V1.Collection.Author? = nil
+        author: PackageCollectionModel.V1.Collection.Author? = nil,
+        metadataProviders: [String: String]? = nil
     ) {
         self.name = name
         self.overview = overview
         self.keywords = keywords
         self.packages = packages
         self.author = author
+        self.metadataProviders = metadataProviders
     }
 }
 
@@ -56,7 +61,8 @@ extension PackageCollectionGeneratorInput: CustomStringConvertible {
             overview=\(self.overview ?? "nil"),
             keywords=\(self.keywords.map { "\($0)" } ?? "nil"),
             packages=\(self.packages),
-            author=\(self.author.map { "\($0)" } ?? "nil")
+            author=\(self.author.map { "\($0)" } ?? "nil"),
+            metadataProviders=\(self.metadataProviders.map { "\($0)" } ?? "nil")
         }
         """
     }

Sources/PackageCollectionGenerator/PackageCollectionGenerate.swift

@@ -51,7 +51,7 @@ public struct PackageCollectionGenerate: ParsableCommand {
     @Option(parsing: .upToNextOption, help:
         """
         Auth tokens each in the format of type:host:token for retrieving additional package metadata via source
-        hosting platform APIs. Currently only GitHub APIs are supported. An example token would be github:github.com:<TOKEN>.
+        hosting platform APIs. Currently GitHub and GitLab APIs are supported. An example token would be github:github.com:<TOKEN>.
         """)
     private var authToken: [String] = []
 
@@ -89,12 +89,37 @@ public struct PackageCollectionGenerate: ParsableCommand {
         let input = try jsonDecoder.decode(PackageCollectionGeneratorInput.self, from: Data(contentsOf: URL(fileURLWithPath: self.inputPath)))
         print("\(input)", verbose: self.verbose)
 
-        let githubPackageMetadataProvider = GitHubPackageMetadataProvider(authTokens: authTokens)
+        // Build the host-to-metadata-provider-mapping
+        var hostToProviderMapping: [String: PackageMetadataProvider] = [
+            "github.com": GitHubPackageMetadataProvider(authTokens: authTokens),
+            "gitlab.com": GitLabPackageMetadataProvider(authTokens: authTokens)
+        ]
+        if let mappingFromInput = input.metadataProviders {
+            for (host, provider) in mappingFromInput {
+                switch provider {
+                case "github":
+                    hostToProviderMapping[host] = GitHubPackageMetadataProvider(authTokens: authTokens)
+                case "gitlab":
+                    hostToProviderMapping[host] = GitLabPackageMetadataProvider(authTokens: authTokens)
+                default:
+                    printError("Provider \(provider) for host \(host) is not implemented")
+                    return
+                }
+            }
+        }
 
         // Generate metadata for each package
         let packages: [Model.Collection.Package] = input.packages.compactMap { package in
             do {
-                let packageMetadata = try self.generateMetadata(for: package, metadataProvider: githubPackageMetadataProvider, jsonDecoder: jsonDecoder)
+                guard let host = package.url.host,
+                      let metadataProvider = hostToProviderMapping[host] else {
+                          printError("Missing provider for host \(package.url.host ?? "invalid host"). Please specify it in the input-json in metadataProviders.")
+                          return nil
+                }
+
+                let packageMetadata = try self.generateMetadata(for: package,
+                                                                   metadataProvider: metadataProvider,
+                                                                   jsonDecoder: jsonDecoder)
                 print("\(packageMetadata)", verbose: self.verbose)
 
                 guard !packageMetadata.versions.isEmpty else {

Sources/PackageCollectionGenerator/PackageMetadataProviders/GitLabPackageManagerMetadataProvider.swift

@@ -0,0 +1,169 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift Package Collection Generator open source project
+//
+// Copyright (c) 2021 Apple Inc. and the Swift Package Collection Generator project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of Swift Package Collection Generator project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Dispatch
+import Foundation
+
+import TSCBasic
+import Basics
+import PackageCollectionsModel
+import Utilities
+
+struct GitLabPackageMetadataProvider: PackageMetadataProvider {
+    private static let apiHostURLPathPostfix = "api/v4"
+
+    private let authTokens: [AuthTokenType: String]
+    private let httpClient: HTTPClient
+    private let decoder: JSONDecoder
+    private enum ResultKeys {
+        case metadata
+    }
+
+    init(authTokens: [AuthTokenType: String] = [:], httpClient: HTTPClient? = nil) {
+        self.authTokens = authTokens
+        self.httpClient = httpClient ?? Self.makeDefaultHTTPClient()
+        self.decoder = JSONDecoder.makeWithDefaults()
+    }
+
+    func get(_ packageURL: URL, callback: @escaping (Result<PackageBasicMetadata, Error>) -> Void) {
+        guard let baseURL: URL = self.apiURL(packageURL) else {
+            return callback(.failure(Errors.invalidGitURL(packageURL)))
+        }
+
+        // get the main data
+        let metadataHeaders = HTTPClientHeaders()
+        let metadataOptions = self.makeRequestOptions(validResponseCodes: [200, 401, 403, 404])
+        let hasAuthorization = metadataOptions.authorizationProvider?(baseURL) != nil
+        var result: Result<HTTPClient.Response, Error> = tsc_await { callback in self.httpClient.get(baseURL, headers: metadataHeaders, options: metadataOptions, completion: callback) }
+
+        if case .success(let response) = result {
+            let apiLimit = response.headers.get("RateLimit-Limit").first.flatMap(Int.init) ?? -1
+            let apiRemaining = response.headers.get("RateLimit-Remaining").first.flatMap(Int.init) ?? -1
+            switch (response.statusCode, hasAuthorization, apiRemaining) {
+            case (_, _, 0):
+                result = .failure(Errors.apiLimitsExceeded(baseURL, apiLimit, apiRemaining))
+            case (401, true, _):
+                result = .failure(Errors.invalidAuthToken(baseURL))
+            case (401, false, _):
+                result = .failure(Errors.permissionDenied(baseURL))
+            case (403, _, _):
+                result = .failure(Errors.permissionDenied(baseURL))
+            case (404, _, _):
+                result = .failure(Errors.notFound(baseURL))
+            case (200, _, _):
+                guard let metadata = try? response.decodeBody(GetProjectResponse.self, using: self.decoder) else {
+                    callback(.failure(Errors.invalidResponse(baseURL, "Invalid body")))
+                    return
+                }
+
+                let license = metadata.license
+                let packageLicense: PackageCollectionModel.V1.License?
+                if let licenseURL = metadata.licenseURL.flatMap({ URL(string: $0) }) {
+                    packageLicense = .init(name: license?.key, url: licenseURL)
+                } else {
+                    packageLicense = nil
+                }
+
+                let model = PackageBasicMetadata(
+                    summary: metadata.description,
+                    keywords: metadata.topics,
+                    readmeURL: metadata.readmeURL.flatMap { URL(string: $0) },
+                    license: packageLicense
+                )
+
+                callback(.success(model))
+            default:
+                callback(.failure(Errors.invalidResponse(baseURL, "Invalid status code: \(response.statusCode)")))
+            }
+        }
+    }
+
+    internal func apiURL(_ url: URL) -> Foundation.URL? {
+        guard let baseURL = URL(string: url.absoluteString.spm_dropGitSuffix()),
+              let urlComponents = URLComponents(url: baseURL, resolvingAgainstBaseURL: false),
+              let scheme = urlComponents.scheme,
+              let host = urlComponents.host else {
+            return nil
+        }
+        let projectPath = urlComponents.path.dropFirst().replacingOccurrences(of: "/", with: "%2F")
+        let apiPrefix = GitLabPackageMetadataProvider.apiHostURLPathPostfix
+        let metadataURL = URL(string: "\(scheme)://\(host)/\(apiPrefix)/projects/\(projectPath)")!
+        return metadataURL
+    }
+
+    private func makeRequestOptions(validResponseCodes: [Int]) -> HTTPClientRequest.Options {
+        var options = HTTPClientRequest.Options()
+        options.addUserAgent = true
+        options.validResponseCodes = validResponseCodes
+        options.authorizationProvider = { url in
+            url.host.flatMap { host in
+                return self.authTokens[.gitlab(host)].flatMap { token in "Bearer \(token)" }
+            }
+        }
+        return options
+    }
+
+    private static func makeDefaultHTTPClient() -> HTTPClient {
+        var client = HTTPClient()
+        client.configuration.requestTimeout = .seconds(2)
+        client.configuration.retryStrategy = .exponentialBackoff(maxAttempts: 3, baseDelay: .milliseconds(50))
+        client.configuration.circuitBreakerStrategy = .hostErrors(maxErrors: 50, age: .seconds(30))
+        return client
+    }
+
+    enum Errors: Error, Equatable {
+        case invalidGitURL(URL)
+        case invalidResponse(URL, String)
+        case permissionDenied(URL)
+        case invalidAuthToken(URL)
+        case apiLimitsExceeded(URL, Int, Int)
+        case notFound(URL)
+    }
+}
+
+extension GitLabPackageMetadataProvider {
+    fileprivate struct GetProjectResponse: Codable {
+        let name: String
+        let fullName: String
+        let description: String?
+        let topics: [String]?
+        let licenseURL: String?
+        let readmeURL: String?
+        let license: License?
+
+        private enum CodingKeys: String, CodingKey {
+            case name
+            case fullName = "name_with_namespace"
+            case description
+            case topics
+            case licenseURL = "license_url"
+            case readmeURL = "readme_url"
+            case license
+        }
+    }
+}
+
+extension GitLabPackageMetadataProvider {
+    fileprivate struct License: Codable {
+        let htmlURL: Foundation.URL
+        let sourceURL: Foundation.URL
+        let key: String?
+
+        private enum CodingKeys: String, CodingKey {
+            case htmlURL = "html_url"
+            case sourceURL = "source_url"
+            case key
+        }
+    }
+}

Sources/PackageCollectionGenerator/PackageMetadataProviders/PackageMetadataProvider.swift

@@ -22,18 +22,23 @@ protocol PackageMetadataProvider {
 
 enum AuthTokenType: Hashable, CustomStringConvertible {
     case github(_ host: String)
+    case gitlab(_ host: String)
 
     var description: String {
         switch self {
         case .github(let host):
             return "github(\(host))"
+        case .gitlab(let host):
+            return "gitlab(\(host))"
         }
     }
 
     static func from(type: String, host: String) -> AuthTokenType? {
         switch type {
         case "github":
             return .github(host)
+        case "gitlab":
+            return .gitlab(host)
         default:
             return nil
         }

Sources/PackageCollectionGenerator/README.md

@@ -47,6 +47,7 @@ Collection metadata:
 * `keywords`: An array of keywords that the collection is associated with. **Optional.**
 * `author`: The author of this package collection. **Optional.**
     * `name`: The author name.
+* `metadataProviders`: A dictionary describing a mapping from hosts to metadata-provider (github / gitlab). **Optional.**
 * `packages`: An array of package objects.
 
 Each item in the `packages` array is a package object with the following fields:
@@ -83,6 +84,9 @@ Each item in the `packages` array is a package object with the following fields:
   ],
   "author": {
     "name": "Jane Doe"
+  },
+  "metadataProviders": {
+    "www.example.com": "gitlab"
   }
 }
 ```

Tests/PackageCollectionGeneratorExecutableTests/PackageCollectionGenerateTests.swift

@@ -64,7 +64,8 @@ final class PackageCollectionGenerateTests: XCTestCase {
                         summary: "Package Baz",
                         versions: ["1.0.0"]
                     ),
-                ]
+                ],
+                metadataProviders: ["package-collection-tests.com": "github"]
             )
             let jsonEncoder = JSONEncoder.makeWithDefaults()
             let inputData = try jsonEncoder.encode(input)
@@ -235,7 +236,8 @@ final class PackageCollectionGenerateTests: XCTestCase {
                         summary: "Package Foo & Bar",
                         excludedVersions: ["0.1.0"]
                     ),
-                ]
+                ],
+                metadataProviders: ["package-collection-tests.com": "github"]
             )
             let jsonEncoder = JSONEncoder.makeWithDefaults()
             let inputData = try jsonEncoder.encode(input)

Tests/PackageCollectionGeneratorTests/Inputs/test-input.json

@@ -17,7 +17,13 @@
       "url": "https://package-collection-tests.com/repos/foobaz.git"
     }
   ],
+  "metadataProviders": {
+      "package-collection-tests.com": "github"
+  },
   "author": {
     "name": "Jane Doe"
+  },
+  "metadataProviders": {
+      "package-collection-tests.com": "github"
   }
 }

Tests/PackageCollectionGeneratorTests/PackageCollectionGeneratorInputTests.swift

@@ -39,7 +39,8 @@ class PackageCollectionGeneratorInputTests: XCTestCase {
                     url: URL(string: "https://package-collection-tests.com/repos/foobaz.git")!
                 ),
             ],
-            author: .init(name: "Jane Doe")
+            author: .init(name: "Jane Doe"),
+            metadataProviders: ["package-collection-tests.com": "github"]
         )
 
         let inputFilePath = AbsolutePath(#file).parentDirectory.appending(components: "Inputs", "test-input.json")
@@ -68,7 +69,8 @@ class PackageCollectionGeneratorInputTests: XCTestCase {
                     readmeURL: URL(string: "https://package-collection-tests.com/repos/foobar/README")!
                 ),
             ],
-            author: .init(name: "Jane Doe")
+            author: .init(name: "Jane Doe"),
+            metadataProviders: ["package-collection-tests.com": "github"]
         )
 
         let data = try JSONEncoder().encode(input)