diff --git a/README.md b/README.md index 844afc93..254b2319 100755 --- a/README.md +++ b/README.md @@ -288,7 +288,7 @@ _Note that for the `producer` mode, the prebuild build phase and `xccc`, `xcld`, | `cache_commit_history` | Number of historical git commits to look for cache artifacts | `10` | ⬜️ | | `source_root` | Source root of the Xcode project | `""` | ⬜️ | | `fingerprint_override_extension` | Fingerprint override extension (sample override `Module.swiftmodule/x86_64.swiftmodule.md5`) | `md5` | ⬜️ | -| `extra_configuration_file` | Configuration file that overrides project configuration | `user.rcinfo` | ⬜️ | +| `extra_configuration_file` | Configuration file that overrides project configuration (this property can be overriden multiple times in different files to chain extra configuration files) | `user.rcinfo` | ⬜️ | | `publishing_sha` | Custom commit sha to publish artifact (producer only) | `nil` | ⬜️ | | `artifact_maximum_age` | Maximum age in days HTTP response should be locally cached before being evicted | `30` | ⬜️ | | `custom_fingerprint_envs` | Extra ENV keys that should be convoluted into the environment fingerprint | `[]` | ⬜️ | diff --git a/Sources/XCRemoteCache/Commands/Libtool/XCLibtoolCreateUniversalBinary.swift b/Sources/XCRemoteCache/Commands/Libtool/XCLibtoolCreateUniversalBinary.swift index 063fd161..d9dcecce 100644 --- a/Sources/XCRemoteCache/Commands/Libtool/XCLibtoolCreateUniversalBinary.swift +++ b/Sources/XCRemoteCache/Commands/Libtool/XCLibtoolCreateUniversalBinary.swift @@ -52,7 +52,7 @@ class XCLibtoolCreateUniversalBinary: XCLibtoolLogic { let config: XCRemoteCacheConfig do { let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath) - config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager) + config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager) .readConfiguration() } catch { errorLog("Libtool initialization failed with error: \(error). Fallbacking to libtool") diff --git a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift index 63761ff5..a0a88d00 100644 --- a/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift +++ b/Sources/XCRemoteCache/Commands/Postbuild/XCPostbuild.swift @@ -34,7 +34,7 @@ public class XCPostbuild { let context: PostbuildContext let cacheHitLogger: CacheHitLogger do { - config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration() + config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration() context = try PostbuildContext(config, env: env) updateProcessTag(context.targetName) let counterFactory: FileStatsCoordinator.CountersFactory = { file, count in diff --git a/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift b/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift index d8246814..78bb8f1c 100644 --- a/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift +++ b/Sources/XCRemoteCache/Commands/Prebuild/XCPrebuild.swift @@ -29,7 +29,7 @@ public class XCPrebuild { let config: XCRemoteCacheConfig let context: PrebuildContext do { - config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration() + config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration() context = try PrebuildContext(config, env: env) updateProcessTag(context.targetName) } catch { diff --git a/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift b/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift index 4f89d457..3788e071 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/Integrate/XCIntegrate.swift @@ -74,7 +74,7 @@ public class XCIntegrate { let binariesDir = commandURL.deletingLastPathComponent() let srcRoot: URL = URL(fileURLWithPath: projectPath).deletingLastPathComponent() - let config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager) + let config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager) .readConfiguration() let context = try IntegrateContext( diff --git a/Sources/XCRemoteCache/Commands/Prepare/XCConfig.swift b/Sources/XCRemoteCache/Commands/Prepare/XCConfig.swift index e7a20661..e271d5de 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/XCConfig.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/XCConfig.swift @@ -32,7 +32,7 @@ public class XCConfig { let fileManager = FileManager.default let config: XCRemoteCacheConfig do { - config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration() + config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration() } catch { exit(1, "FATAL: Prepare initialization failed with error: \(error)") } diff --git a/Sources/XCRemoteCache/Commands/Prepare/XCPrepare.swift b/Sources/XCRemoteCache/Commands/Prepare/XCPrepare.swift index fb998eb2..b6046504 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/XCPrepare.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/XCPrepare.swift @@ -61,7 +61,7 @@ public class XCPrepare { var context: PrepareContext let xcodeVersion: String do { - config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration() + config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration() context = try PrepareContext(config, offline: offline) xcodeVersion = try customXcodeBuildNumber ?? XcodeProbeImpl(shell: shellGetStdout).read().buildVersion } catch { diff --git a/Sources/XCRemoteCache/Commands/Prepare/XCPrepareMark.swift b/Sources/XCRemoteCache/Commands/Prepare/XCPrepareMark.swift index 298c4fce..4351bf6e 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/XCPrepareMark.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/XCPrepareMark.swift @@ -46,7 +46,7 @@ public class XCPrepareMark { let context: PrepareMarkContext let xcodeVersion: String do { - config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration() + config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration() context = try PrepareMarkContext(config) xcodeVersion = try xcode ?? XcodeProbeImpl(shell: shellGetStdout).read().buildVersion } catch { diff --git a/Sources/XCRemoteCache/Commands/Prepare/XCStats.swift b/Sources/XCRemoteCache/Commands/Prepare/XCStats.swift index f505258b..c12bd01c 100644 --- a/Sources/XCRemoteCache/Commands/Prepare/XCStats.swift +++ b/Sources/XCRemoteCache/Commands/Prepare/XCStats.swift @@ -36,7 +36,7 @@ public class XCStats { let config: XCRemoteCacheConfig let context: XCStatsContext do { - config = try XCRemoteCacheConfigReader(env: env, fileManager: fileManager).readConfiguration() + config = try XCRemoteCacheConfigReader(env: env, fileReader: fileManager).readConfiguration() try context = XCStatsContext(config, fileManager: fileManager) } catch { exit(1, "FATAL: Prepare initialization failed with error: \(error)") diff --git a/Sources/XCRemoteCache/Commands/ProductBinaryCreator/XCCreateBinary.swift b/Sources/XCRemoteCache/Commands/ProductBinaryCreator/XCCreateBinary.swift index d1b103cf..70ed2ccc 100644 --- a/Sources/XCRemoteCache/Commands/ProductBinaryCreator/XCCreateBinary.swift +++ b/Sources/XCRemoteCache/Commands/ProductBinaryCreator/XCCreateBinary.swift @@ -70,7 +70,7 @@ public class XCCreateBinary { let config: XCRemoteCacheConfig do { let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath) - config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager) + config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager) .readConfiguration() } catch { errorLog("\(stepDescription) initialization failed with error: \(error). Fallbacking to \(fallbackCommand)") diff --git a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift index 73976ae9..2d988eea 100644 --- a/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift +++ b/Sources/XCRemoteCache/Commands/Swiftc/XCSwiftc.swift @@ -70,7 +70,7 @@ public class XCSwiftc { let context: SwiftcContext do { let srcRoot: URL = URL(fileURLWithPath: fileManager.currentDirectoryPath) - config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileManager: fileManager) + config = try XCRemoteCacheConfigReader(srcRootPath: srcRoot.path, fileReader: fileManager) .readConfiguration() context = try SwiftcContext(config: config, input: inputArgs) } catch { diff --git a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift index 0b1dc62c..be5bc27f 100644 --- a/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift +++ b/Sources/XCRemoteCache/Config/XCRemoteCacheConfig.swift @@ -316,31 +316,41 @@ class XCRemoteCacheConfigReader { /// Name of the configuration file, required in $(SRCROOT) location private static let configurationFile = ".rcinfo" private let srcRoot: String - private let fileManager: FileManager + private let fileReader: FileReader private lazy var yamlDecorer = YAMLDecoder(encoding: .utf8) - init(env: [String: String], fileManager: FileManager) throws { + init(env: [String: String], fileReader: FileReader) throws { let explicitSrcRoot: String? = env.readEnv(key: "SRCROOT") - srcRoot = explicitSrcRoot ?? fileManager.currentDirectoryPath - self.fileManager = fileManager + srcRoot = explicitSrcRoot ?? FileManager.default.currentDirectoryPath + self.fileReader = fileReader } - init(srcRootPath srcRoot: String, fileManager: FileManager) { + init(srcRootPath srcRoot: String, fileReader: FileReader) { self.srcRoot = srcRoot - self.fileManager = fileManager + self.fileReader = fileReader } + // Reads the final configuration by loading all extra configs + // until reaching a config that doesn't override `extraConfigurationFile` func readConfiguration() throws -> XCRemoteCacheConfig { let rootURL = URL(fileURLWithPath: srcRoot) let configURL = URL(fileURLWithPath: Self.configurationFile, relativeTo: rootURL) let userConfigs = try readUserConfig(configURL) var config = XCRemoteCacheConfig(sourceRoot: srcRoot).merged(with: userConfigs) - let extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL) - do { - let extraConfig = try readUserConfig(extraConfURL) - config = config.merged(with: extraConfig) - } catch { - infoLog("Extra config override failed with \(error). Skipping extra configuration") + var extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL) + var visitedFiles = Set([configURL]) + while !visitedFiles.contains(extraConfURL) { + do { + let extraConfig = try readUserConfig(extraConfURL) + debugLog("Reading extra configuration from \(extraConfURL)") + config = config.merged(with: extraConfig) + visitedFiles.insert(extraConfURL) + // Advance extra configuration + extraConfURL = URL(fileURLWithPath: config.extraConfigurationFile, relativeTo: rootURL) + } catch { + infoLog("Extra config override failed with \(error). Skipping extra configuration") + break + } } return try config.verifyAndApplyDefaults() @@ -348,7 +358,7 @@ class XCRemoteCacheConfigReader { /// Reads user configuration from a file private func readUserConfig(_ file: URL) throws -> ConfigFileScheme { - let configurationContent = fileManager.contents(atPath: file.path) + let configurationContent = try fileReader.contents(atPath: file.path) guard let configurationData = configurationContent else { throw XCRemoteCacheConfigReaderError.missingConfigurationFile(file) } diff --git a/Tests/XCRemoteCacheTests/Config/XCRemoteCacheConfigReaderTests.swift b/Tests/XCRemoteCacheTests/Config/XCRemoteCacheConfigReaderTests.swift new file mode 100644 index 00000000..326580cf --- /dev/null +++ b/Tests/XCRemoteCacheTests/Config/XCRemoteCacheConfigReaderTests.swift @@ -0,0 +1,88 @@ +// Copyright (c) 2022 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +@testable import XCRemoteCache +import XCTest + +class XCRemoteCacheConfigReaderTests: XCTestCase { + + private var fileReader: FileAccessorFake! + private var reader: XCRemoteCacheConfigReader! + + override func setUp() { + super.setUp() + fileReader = FileAccessorFake(mode: .normal) + reader = XCRemoteCacheConfigReader(srcRootPath: "/", fileReader: fileReader) + } + + func testReadsFromExtraConfig() throws { + try fileReader.write(toPath: "/.rcinfo", contents: "cache_addresses: [test]") + + let config = try reader.readConfiguration() + + XCTAssertEqual(config.cacheAddresses, ["test"]) + } + + func testOverridesExtraConfigFromExtraFile() throws { + try fileReader.write(toPath: "/.rcinfo", contents: "cache_addresses: [test]") + try fileReader.write(toPath: "/user.rcinfo", contents: "cache_addresses: [user]") + + let config = try reader.readConfiguration() + + XCTAssertEqual(config.cacheAddresses, ["user"]) + } + + func testReadsExtraConfigMultipleTimes() throws { + try fileReader.write(toPath: "/.rcinfo", contents: "cache_addresses: [test]") + try fileReader.write(toPath: "/user.rcinfo", contents: """ + cache_addresses: [user] + extra_configuration_file: user2.rcinfo + """) + try fileReader.write(toPath: "/user2.rcinfo", contents: "cache_addresses: [user2]") + + let config = try reader.readConfiguration() + + XCTAssertEqual(config.cacheAddresses, ["user2"]) + } + + func testBreaksImportingExtraConfigIfReachingALoop() throws { + try fileReader.write(toPath: "/.rcinfo", contents: "cache_addresses: [test]") + try fileReader.write(toPath: "/user.rcinfo", contents: """ + cache_addresses: [user] + extra_configuration_file: .rcinfo + """) + + let config = try reader.readConfiguration() + + XCTAssertEqual(config.cacheAddresses, ["user"]) + } + + func testBreaksImportingExtraConfigIfFileDoesntExist() throws { + try fileReader.write(toPath: "/.rcinfo", contents: "cache_addresses: [test]") + try fileReader.write(toPath: "/user.rcinfo", contents: """ + cache_addresses: [user] + extra_configuration_file: nonexisting.rcinfo + """) + + let config = try reader.readConfiguration() + + XCTAssertEqual(config.cacheAddresses, ["user"]) + XCTAssertEqual(config.extraConfigurationFile, "nonexisting.rcinfo") + } +}