From cc0af1d4f83aafd8dfbc38b952a99f315d17e74f Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 5 Jun 2025 14:44:03 -0700 Subject: [PATCH] [CAS] Allow uncached job from CAS based dependency scanning Create a path that swift-frontend can execute an uncached job from modules built with CAS based explicit module build. The new flag -import-module-from-cas will allow an uncached build to load module from CAS, and combined with source file from real file system to build the current module. This allows quick iterations that bypasses CAS, without full dependency scanning every time in between. rdar://152441866 --- include/swift/Basic/CASOptions.h | 3 + include/swift/Frontend/Frontend.h | 3 +- include/swift/Option/FrontendOptions.td | 3 + lib/ClangImporter/ClangImporter.cpp | 2 +- lib/Frontend/CompilerInvocation.cpp | 2 + lib/Frontend/Frontend.cpp | 3 +- test/CAS/uncached-casfs.swift | 79 +++++++++++++++++++++++++ 7 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 test/CAS/uncached-casfs.swift diff --git a/include/swift/Basic/CASOptions.h b/include/swift/Basic/CASOptions.h index 82f7731b88dd4..567c34671fc49 100644 --- a/include/swift/Basic/CASOptions.h +++ b/include/swift/Basic/CASOptions.h @@ -34,6 +34,9 @@ class CASOptions final { /// Skip replaying outputs from cache. bool CacheSkipReplay = false; + /// Import modules from CAS. + bool ImportModuleFromCAS = false; + /// CASOptions clang::CASOptions CASOpts; diff --git a/include/swift/Frontend/Frontend.h b/include/swift/Frontend/Frontend.h index 55d984a53db8a..029e296ab5d38 100644 --- a/include/swift/Frontend/Frontend.h +++ b/include/swift/Frontend/Frontend.h @@ -176,7 +176,8 @@ class CompilerInvocation { } bool requiresCAS() const { - return CASOpts.EnableCaching || IRGenOpts.UseCASBackend; + return CASOpts.EnableCaching || IRGenOpts.UseCASBackend || + CASOpts.ImportModuleFromCAS; } void setClangModuleCachePath(StringRef Path) { diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index cef1ba7eadbf0..e22aa30ee6605 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -244,6 +244,9 @@ def module_can_import_version: MultiArg<["-"], "module-can-import-version", 3>, MetaVarName<" ">, HelpText<"Specify canImport module and versions">; +def module_import_from_cas: Flag<["-"], "module-import-from-cas">, + HelpText<"Import modules from CAS instead of file system">; + def disable_cross_import_overlay_search: Flag<["-"], "disable-cross-import-overlay-search">, HelpText<"Disable searching for cross import overlay file">; diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index 5af12a07e8d34..9a4f9eb915d5c 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -1181,7 +1181,7 @@ std::optional> ClangImporter::getClangCC1Arguments( // to reduce the number of argument passing on the command-line and swift // compiler can be more efficient to compute swift cache key without having // the knowledge about clang command-line options. - if (ctx.CASOpts.EnableCaching) { + if (ctx.CASOpts.EnableCaching || ctx.CASOpts.ImportModuleFromCAS) { CI->getCASOpts() = ctx.CASOpts.CASOpts; // When clangImporter is used to compile (generate .pcm or .pch), need to // inherit the include tree from swift args (last one wins) and clear the diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index f25e0cba01b31..38ea46f28c3c4 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -798,6 +798,8 @@ static bool ParseCASArgs(CASOptions &Opts, ArgList &Args, std::string(Value)); } + Opts.ImportModuleFromCAS |= Args.hasArg(OPT_module_import_from_cas); + if (auto *A = Args.getLastArg(OPT_clang_include_tree_root)) Opts.ClangIncludeTree = A->getValue(); if (auto *A = Args.getLastArg(OPT_clang_include_tree_filelist)) diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index 17527cffd3a20..84b2a19c384ee 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -810,7 +810,8 @@ bool CompilerInstance::setUpModuleLoaders() { if (ExplicitModuleBuild || !Invocation.getSearchPathOptions().ExplicitSwiftModuleMapPath.empty() || !Invocation.getSearchPathOptions().ExplicitSwiftModuleInputs.empty()) { - if (Invocation.getCASOptions().EnableCaching) + if (Invocation.getCASOptions().EnableCaching || + Invocation.getCASOptions().ImportModuleFromCAS) ESML = ExplicitCASModuleLoader::create( *Context, getObjectStore(), getActionCache(), getDependencyTracker(), MLM, Invocation.getSearchPathOptions().ExplicitSwiftModuleMapPath, diff --git a/test/CAS/uncached-casfs.swift b/test/CAS/uncached-casfs.swift new file mode 100644 index 0000000000000..7369cdd6da1d7 --- /dev/null +++ b/test/CAS/uncached-casfs.swift @@ -0,0 +1,79 @@ +// RUN: %empty-directory(%t) +// RUN: split-file %s %t + +// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O -module-cache-path %t/clang-module-cache \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: -import-objc-header %t/base/Bridging.h -scanner-output-dir %t -auto-bridging-header-chaining -scanner-debug-write-output \ +// RUN: %t/base/test.swift %t/base/foo.swift -I %t/include -o %t/deps.json -cache-compile-job -cas-path %t/cas + +// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/shim.cmd +// RUN: %swift_frontend_plain @%t/shim.cmd +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:Dummy > %t/dummy.cmd +// RUN: %swift_frontend_plain @%t/dummy.cmd + +// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json bridgingHeader > %t/header.cmd +// RUN: %target-swift-frontend @%t/header.cmd -disable-implicit-swift-modules -O -o %t/bridging.pch +// RUN: %cache-tool -cas-path %t/cas -cache-tool-action print-output-keys -- \ +// RUN: %target-swift-frontend @%t/header.cmd -disable-implicit-swift-modules -O -o %t/bridging.pch > %t/keys.json +// RUN: %{python} %S/Inputs/ExtractOutputKey.py %t/keys.json > %t/key + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd +// RUN: echo "\"-disable-implicit-string-processing-module-import\"" >> %t/MyApp.cmd +// RUN: echo "\"-disable-implicit-concurrency-module-import\"" >> %t/MyApp.cmd +// RUN: echo "\"-disable-implicit-swift-modules\"" >> %t/MyApp.cmd +// RUN: echo "\"-import-objc-header\"" >> %t/MyApp.cmd +// RUN: echo "\"%t/base/Bridging.h\"" >> %t/MyApp.cmd +// RUN: echo "\"-import-pch\"" >> %t/MyApp.cmd +// RUN: echo "\"%t/bridging.pch\"" >> %t/MyApp.cmd +// RUN: echo "\"-bridging-header-pch-key\"" >> %t/MyApp.cmd +// RUN: echo "\"@%t/key\"" >> %t/MyApp.cmd +// RUN: echo "\"-explicit-swift-module-map-file\"" >> %t/MyApp.cmd +// RUN: echo "\"@%t/map.casid\"" >> %t/MyApp.cmd + +// RUN: sed -e "s@VFS_DIR@%{/t:regex_replacement}/base@g" -e "s@EXTERNAL_DIR@%{/t:regex_replacement}/modified@g" %t/base.yaml > %t/overlay.yaml + +// RUN: %target-swift-frontend %t/base/test.swift %t/base/foo.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c \ +// RUN: -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd -vfsoverlay %t/overlay.yaml -module-import-from-cas + + +//--- base/test.swift +import Dummy +public func testFunc() { + non_existing_func() +} + +//--- base/foo.swift +public func foo() {} + +//--- modified/test.swift +import Dummy +public func testFunc() { + dummy() + bridge() +} + +//--- base/Bridging.h +void bridge(void); + +//--- include/module.modulemap +module Dummy { + umbrella header "Dummy.h" +} + +//--- include/Dummy.h +void dummy(void); + +//--- base.yaml +{ + version: 0, + roots: [ + { + type: "directory-remap", + name: "VFS_DIR", + external-contents: "EXTERNAL_DIR" + } + ] +}