Merge pull request #117 from polac24/20220411-loop-configs

Load extra configuration in a chain

README.md

@@ -288,7 +288,7 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`,
 | `cache_commit_history` | Number of historical git commits to look for cache artifacts | `10` | ⬜️ |
 | `source_root` | Source root of the Xcode project | `""` | ⬜️ |
 | `fingerprint_override_extension` | Fingerprint override extension (sample override `Module.swiftmodule/x86_64.swiftmodule.md5`) | `md5` | ⬜️ |
-| `extra_configuration_file` | Configuration file that overrides project configuration | `user.rcinfo` | ⬜️ |
+| `extra_configuration_file` | Configuration file that overrides project configuration (this property can be overriden multiple times in different files to chain extra configuration files) | `user.rcinfo` | ⬜️ |
 | `publishing_sha` | Custom commit sha to publish artifact (producer only) | `nil` | ⬜️ |
 | `artifact_maximum_age` | Maximum age in days HTTP response should be locally cached before being evicted | `30` | ⬜️ |
 | `custom_fingerprint_envs` | Extra ENV keys that should be convoluted into the environment fingerprint | `[]` | ⬜️ |

Sources/XCRemoteCache/Commands/Libtool/XCLibtoolCreateUniversalBinary.swift

@@ -52,7 +52,7 @@ class XCLibtoolCreateUniversalBinary: XCLibtoolLogic {
         let config: XCRemoteCacheConfig
         do {
             let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
-            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
+            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
                 .readConfiguration()
         } catch {
             errorLog("Libtool initialization failed with error: \(error). Fallbacking to libtool")

Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift

@@ -34,7 +34,7 @@ public class XCPostbuild {
         let context: PostbuildContext
         let cacheHitLogger: CacheHitLogger
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             context = try PostbuildContext(config, env: env)
             updateProcessTag(context.targetName)
             let counterFactory: FileStatsCoordinator.CountersFactory = { file, count in

Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift

@@ -29,7 +29,7 @@ public class XCPrebuild {
         let config: XCRemoteCacheConfig
         let context: PrebuildContext
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             context = try PrebuildContext(config, env: env)
             updateProcessTag(context.targetName)
         } catch {

Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift

@@ -74,7 +74,7 @@ public class XCIntegrate {
             let binariesDir = commandURL.deletingLastPathComponent()
 
             let srcRoot: URL = URL(fileURLWithPath: projectPath).deletingLastPathComponent()
-            let config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
+            let config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
                 .readConfiguration()
 
             let context = try IntegrateContext(

Sources/XCRemoteCache/Commands/Prepare/XCConfig.swift

@@ -32,7 +32,7 @@ public class XCConfig {
         let fileManager = FileManager.default
         let config: XCRemoteCacheConfig
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
         } catch {
             exit(1, "FATAL: Prepare initialization failed with error: \(error)")
         }

Sources/XCRemoteCache/Commands/Prepare/XCPrepare.swift

@@ -61,7 +61,7 @@ public class XCPrepare {
         var context: PrepareContext
         let xcodeVersion: String
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             context = try PrepareContext(config, offline: offline)
             xcodeVersion = try customXcodeBuildNumber ?? XcodeProbeImpl(shell: shellGetStdout).read().buildVersion
         } catch {

Sources/XCRemoteCache/Commands/Prepare/XCPrepareMark.swift

@@ -46,7 +46,7 @@ public class XCPrepareMark {
         let context: PrepareMarkContext
         let xcodeVersion: String
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             context = try PrepareMarkContext(config)
             xcodeVersion = try xcode ?? XcodeProbeImpl(shell: shellGetStdout).read().buildVersion
         } catch {

Sources/XCRemoteCache/Commands/Prepare/XCStats.swift

@@ -36,7 +36,7 @@ public class XCStats {
         let config: XCRemoteCacheConfig
         let context: XCStatsContext
         do {
-            config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration()
+            config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration()
             try context = XCStatsContext(config, fileManager: fileManager)
         } catch {
             exit(1, "FATAL: Prepare initialization failed with error: \(error)")

Sources/XCRemoteCache/Commands/ProductBinaryCreator/XCCreateBinary.swift

@@ -70,7 +70,7 @@ public class XCCreateBinary {
         let config: XCRemoteCacheConfig
         do {
             let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
-            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
+            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
                 .readConfiguration()
         } catch {
             errorLog("\(stepDescription) initialization failed with error: \(error). Fallbacking to \(fallbackCommand)")

Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift

@@ -70,7 +70,7 @@ public class XCSwiftc {
         let context: SwiftcContext
         do {
             let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath)
-            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager)
+            config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager)
                 .readConfiguration()
             context = try SwiftcContext(config: config, input: inputArgs)
         } catch {

Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift

@@ -316,39 +316,49 @@ class XCRemoteCacheConfigReader {
     /// Name of the configuration file, required in $(SRCROOT) location
     private static let configurationFile = ".rcinfo"
     private let srcRoot: String
-    private let fileManager: FileManager
+    private let fileReader: FileReader
     private lazy var yamlDecorer = YAMLDecoder(encoding: .utf8)
 
-    init(env: [String: String], fileManager: FileManager) throws {
+    init(env: [String: String], fileReader: FileReader) throws {
         let explicitSrcRoot: String? = env.readEnv(key: "SRCROOT")
-        srcRoot = explicitSrcRoot ?? fileManager.currentDirectoryPath
-        self.fileManager = fileManager
+        srcRoot = explicitSrcRoot ?? FileManager.default.currentDirectoryPath
+        self.fileReader = fileReader
     }
 
-    init(srcRootPath srcRoot: String, fileManager: FileManager) {
+    init(srcRootPath srcRoot: String, fileReader: FileReader) {
         self.srcRoot = srcRoot
-        self.fileManager = fileManager
+        self.fileReader = fileReader
     }
 
+    // Reads the final configuration by loading all extra configs
+    // until reaching a config that doesn't override `extraConfigurationFile`
     func readConfiguration() throws -> XCRemoteCacheConfig {
         let rootURL = URL(fileURLWithPath: srcRoot)
         let configURL = URL(fileURLWithPath: Self.configurationFile, relativeTo: rootURL)
         let userConfigs = try readUserConfig(configURL)
         var config = XCRemoteCacheConfig(sourceRoot: srcRoot).merged(with: userConfigs)
-        let extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL)
-        do {
-            let extraConfig = try readUserConfig(extraConfURL)
-            config = config.merged(with: extraConfig)
-        } catch {
-            infoLog("Extra config override failed with \(error). Skipping extra configuration")
+        var extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL)
+        var visitedFiles = Set([configURL])
+        while !visitedFiles.contains(extraConfURL) {
+            do {
+                let extraConfig = try readUserConfig(extraConfURL)
+                debugLog("Reading extra configuration from \(extraConfURL)")
+                config = config.merged(with: extraConfig)
+                visitedFiles.insert(extraConfURL)
+                // Advance extra configuration
+                extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL)
+            } catch {
+                infoLog("Extra config override failed with \(error). Skipping extra configuration")
+                break
+            }
         }
 
         return try config.verifyAndApplyDefaults()
     }
 
     /// Reads user configuration from a file
     private func readUserConfig(_ file: URL) throws -> ConfigFileScheme {
-        let configurationContent = fileManager.contents(atPath: file.path)
+        let configurationContent = try fileReader.contents(atPath: file.path)
         guard let configurationData = configurationContent else {
             throw XCRemoteCacheConfigReaderError.missingConfigurationFile(file)
         }

Tests/XCRemoteCacheTests/Config/XCRemoteCacheConfigReaderTests.swift

@@ -0,0 +1,88 @@
+// Copyright (c) 2022 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+@testable import XCRemoteCache
+import XCTest
+
+class XCRemoteCacheConfigReaderTests: XCTestCase {
+
+    private var fileReader: FileAccessorFake!
+    private var reader: XCRemoteCacheConfigReader!
+
+    override func setUp() {
+        super.setUp()
+        fileReader = FileAccessorFake(mode: .normal)
+        reader = XCRemoteCacheConfigReader(srcRootPath: "/", fileReader: fileReader)
+    }
+
+    func testReadsFromExtraConfig() throws {
+        try fileReader.write(toPath: "/.rcinfo", contents: "cache_addresses: [test]")
+
+        let config = try reader.readConfiguration()
+
+        XCTAssertEqual(config.cacheAddresses, ["test"])
+    }
+
+    func testOverridesExtraConfigFromExtraFile() throws {
+        try fileReader.write(toPath: "/.rcinfo", contents: "cache_addresses: [test]")
+        try fileReader.write(toPath: "/user.rcinfo", contents: "cache_addresses: [user]")
+
+        let config = try reader.readConfiguration()
+
+        XCTAssertEqual(config.cacheAddresses, ["user"])
+    }
+
+    func testReadsExtraConfigMultipleTimes() throws {
+        try fileReader.write(toPath: "/.rcinfo", contents: "cache_addresses: [test]")
+        try fileReader.write(toPath: "/user.rcinfo", contents: """
+        cache_addresses: [user]
+        extra_configuration_file: user2.rcinfo
+        """)
+        try fileReader.write(toPath: "/user2.rcinfo", contents: "cache_addresses: [user2]")
+
+        let config = try reader.readConfiguration()
+
+        XCTAssertEqual(config.cacheAddresses, ["user2"])
+    }
+
+    func testBreaksImportingExtraConfigIfReachingALoop() throws {
+        try fileReader.write(toPath: "/.rcinfo", contents: "cache_addresses: [test]")
+        try fileReader.write(toPath: "/user.rcinfo", contents: """
+        cache_addresses: [user]
+        extra_configuration_file: .rcinfo
+        """)
+
+        let config = try reader.readConfiguration()
+
+        XCTAssertEqual(config.cacheAddresses, ["user"])
+    }
+
+    func testBreaksImportingExtraConfigIfFileDoesntExist() throws {
+        try fileReader.write(toPath: "/.rcinfo", contents: "cache_addresses: [test]")
+        try fileReader.write(toPath: "/user.rcinfo", contents: """
+        cache_addresses: [user]
+        extra_configuration_file: nonexisting.rcinfo
+        """)
+
+        let config = try reader.readConfiguration()
+
+        XCTAssertEqual(config.cacheAddresses, ["user"])
+        XCTAssertEqual(config.extraConfigurationFile, "nonexisting.rcinfo")
+    }
+}