From 8f9c5552670a4e42443828c2e266ca5080b4eeb6 Mon Sep 17 00:00:00 2001
From: MerlynOMsft <44985659+merlynomsft@users.noreply.github.com>
Date: Thu, 3 Oct 2024 17:05:21 -0700
Subject: [PATCH] Speed up buildconfig generation 6x (6m->1m)

---
 BuildConfigGen/GitUtil.cs | 70 +++++++++++++++++++++++++++++----------
 BuildConfigGen/Program.cs |  5 ++-
 2 files changed, 57 insertions(+), 18 deletions(-)

diff --git a/BuildConfigGen/GitUtil.cs b/BuildConfigGen/GitUtil.cs
index f4863e92228..a8cdfc516ef 100644
--- a/BuildConfigGen/GitUtil.cs
+++ b/BuildConfigGen/GitUtil.cs
@@ -4,6 +4,7 @@ namespace BuildConfigGen
 {
     internal static class GitUtil
     {
+        static (string? gitRoot, IEnumerable<string>? cache) NonIgnoredFileListCache;
 
         public static void GetUntrackedFiles(string taskTarget, out IEnumerable<string> toAdd, out IEnumerable<string> toRemove)
         {
@@ -122,40 +123,75 @@ internal static string GetGitRootPath(string currentDir)
 
         internal static IEnumerable<string> GetNonIgnoredFileListFromPath(string gitRoot, string taskTarget)
         {
-            string gitIgnorePathBak = Path.Combine(gitRoot, ".gitignore.bak");
-            string gitIgnore = Path.Combine(gitRoot, ".gitignore");
+            // the new version gets the non-ignored files frmo the root, caches and filters based on taskTarget.  It's 6x faster
+            return GetNonIgnoredFileListFromPathInnerIncludingFromGeneratedLocalFilteredByTaskTarget(gitRoot, taskTarget);
+        }
 
-            bool needsGitIgnoreUpdate = taskTarget.Contains("/_generated_local/") || taskTarget.Contains(@"\_generated_local\");
+        private static IEnumerable<string> GetNonIgnoredFileListFromPathInnerIncludingFromGeneratedLocalFilteredByTaskTarget(string gitRoot, string taskTarget)
+        {
+            if (!taskTarget.StartsWith(gitRoot, StringComparison.OrdinalIgnoreCase))
+            {
+                throw new Exception($"expected taskTarget={taskTarget} to start with gitRoot={gitRoot}");
+            }
 
-            string? gitIgnoreContent = null;
+            string taskTargetSubpath = taskTarget.Substring(gitRoot.Length);
 
-            if (needsGitIgnoreUpdate)
+            if(NonIgnoredFileListCache.gitRoot is not null)
             {
-                gitIgnoreContent = File.ReadAllText(gitIgnore);
-                const string genertedLocalPath = "_generated_local/";
+                NonIgnoredFileListCache.gitRoot = gitRoot;
 
-                if (!gitIgnoreContent.Contains(genertedLocalPath))
+                if(gitRoot!= NonIgnoredFileListCache.gitRoot)
                 {
-                    throw new Exception("Expected " + genertedLocalPath + " in " + gitIgnore);
+                    throw new Exception($"BUG: gitroot={gitRoot} expected to match {NonIgnoredFileListCache.gitRoot}=NonIgnoredFileListCache.gitRoot");
                 }
+            }
 
-                gitIgnoreContent = gitIgnoreContent.Replace(genertedLocalPath, "");
-
-                File.Copy(gitIgnore, gitIgnorePathBak, true);
+            if (NonIgnoredFileListCache.cache is null)
+            {
+                NonIgnoredFileListCache.cache = GetNonIgnoredFileListFromPathInnerIncludingFromGeneratedLocal(gitRoot);
             }
 
-            try
+            return NonIgnoredFileListCache.cache.Where(x => x.StartsWith(taskTargetSubpath + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase)).Select(x =>
             {
-                if (needsGitIgnoreUpdate)
+                var y = x.Substring(taskTargetSubpath.Length);
+
+                if (y.StartsWith("/") || y.StartsWith(@"\"))
+                {
+                    return y.Substring(1);
+                }
+                else
                 {
-                    File.WriteAllText(gitIgnore, gitIgnoreContent);
+                    throw new Exception($"BUG: expected y={y} to start with / or \\ ");
                 }
+            });
+        }
+
+        private static IEnumerable<string> GetNonIgnoredFileListFromPathInnerIncludingFromGeneratedLocal(string gitRoot)
+        {
+            string gitIgnorePathBak = Path.Combine(gitRoot, ".gitignore.bak");
+            string gitIgnore = Path.Combine(gitRoot, ".gitignore");
+
+            string? gitIgnoreContent = File.ReadAllText(gitIgnore);
+            const string genertedLocalPath = "_generated_local/";
+
+            if (!gitIgnoreContent.Contains(genertedLocalPath))
+            {
+                throw new Exception("Expected " + genertedLocalPath + " in " + gitIgnore);
+            }
+
+            gitIgnoreContent = gitIgnoreContent.Replace(genertedLocalPath, "");
+
+            File.Copy(gitIgnore, gitIgnorePathBak, true);
+
+            try
+            {
+                File.WriteAllText(gitIgnore, gitIgnoreContent);
 
-                return GetNonIgnoredFileListFromPathInner(taskTarget);
+                return GetNonIgnoredFileListFromPathInner(gitRoot);
             }
             finally
             {
-                if (needsGitIgnoreUpdate)
+                if (File.Exists(gitIgnorePathBak))
                 {
                     File.Move(gitIgnorePathBak, gitIgnore, true);
                 }
diff --git a/BuildConfigGen/Program.cs b/BuildConfigGen/Program.cs
index 5e58043401a..a20ffbb22be 100644
--- a/BuildConfigGen/Program.cs
+++ b/BuildConfigGen/Program.cs
@@ -73,8 +73,11 @@ static void Main(string? task = null, string? configs = null, int? currentSprint
         {
             try
             {
+                Stopwatch sw = new Stopwatch();
+                sw.Start();
                 ensureUpdateModeVerifier = new EnsureUpdateModeVerifier(!writeUpdates);
                 MainInner(task, configs, currentSprint, writeUpdates, allTasks, getTaskVersionTable, debugAgentDir, includeLocalPackagesBuildConfig);
+                Console.WriteLine("Elapsed=" + sw.ToString());
             }
             catch (Exception e2)
             {
@@ -945,7 +948,7 @@ private static void CopyConfig(string gitRootPath, string taskTargetPathOrUnders
             HashSet<string> pathsToRemoveFromOutput;
 
             // In case if task was not generated yet, we don't need to get the list of files to remove, because taskOutput not exists yet
-            if (Directory.Exists(taskOutput) && !config.useAltGeneratedPath /* exclude alt which is .gitignore */)
+            if (Directory.Exists(taskOutput))
             {
                 pathsToRemoveFromOutput = new HashSet<string>(GitUtil.GetNonIgnoredFileListFromPath(gitRootPath, taskOutput));
             }