Automatically cleanup old artifacts of file-based apps

documentation/general/dotnet-run-file.md

@@ -184,9 +184,12 @@ The subdirectory is created by the SDK CLI with permissions restricting access t
 Note that it is possible for multiple users to run the same file-based program, however each user's run uses different build artifacts since the base directory is unique per user.
 Apart from keeping the source directory clean, such artifact isolation also avoids clashes of build outputs that are not project-scoped, like `project.assets.json`, in the case of multiple entry-point files.
 
-Artifacts are cleaned periodically by a background task that is started by `dotnet run` and
-removes current user's `dotnet run` build outputs that haven't been used in some time.
+Artifacts are cleaned periodically (every 2 days) by a background task that is started by `dotnet run` and
+removes current user's `dotnet run` build outputs that haven't been used in 30 days.
 They are not cleaned immediately because they can be re-used on subsequent runs for better performance.
+The automatic cleanup can be disabled by environment variable `DOTNET_CLI_DISABLE_FILE_BASED_APP_ARTIFACTS_AUTOMATIC_CLEANUP=true`,
+but other parameters of the automatic cleanup are currently not configurable.
+The same cleanup can be performed manually via command `dotnet clean-file-based-app-artifacts`.
 
 ## Directives for project metadata
 

src/Cli/dotnet/Commands/Clean/CleanCommand.cs

@@ -31,7 +31,7 @@ public static CommandBase FromParseResult(ParseResult result, string? msbuildPat
                 NoBuild = false,
                 NoRestore = true,
                 NoCache = true,
-                NoBuildMarkers = true,
+                NoWriteBuildMarkers = true,
             },
             static (msbuildArgs, msbuildPath) => new CleanCommand(msbuildArgs, msbuildPath),
             [ CommonOptions.PropertiesOption, CommonOptions.RestorePropertiesOption, CleanCommandParser.TargetOption ],

src/Cli/dotnet/Commands/Clean/CleanCommandParser.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.CommandLine;
+using Microsoft.DotNet.Cli.Commands.Clean.FileBasedAppArtifacts;
 using Microsoft.DotNet.Cli.Extensions;
 
 namespace Microsoft.DotNet.Cli.Commands.Clean;
@@ -59,6 +60,7 @@ private static Command ConstructCommand()
         command.Options.Add(NoLogoOption);
         command.Options.Add(CommonOptions.DisableBuildServersOption);
         command.Options.Add(TargetOption);
+        command.Subcommands.Add(CleanFileBasedAppArtifactsCommandParser.Command);
 
         command.SetAction(CleanCommand.Run);
 

src/Cli/dotnet/Commands/Clean/FileBasedAppArtifacts/CleanFileBasedAppArtifactsCommand.cs

@@ -0,0 +1,190 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.CommandLine;
+using System.Diagnostics;
+using System.Text.Json;
+using Microsoft.DotNet.Cli.Commands.Run;
+using Microsoft.DotNet.Cli.Utils;
+using Microsoft.DotNet.Cli.Utils.Extensions;
+
+namespace Microsoft.DotNet.Cli.Commands.Clean.FileBasedAppArtifacts;
+
+internal sealed class CleanFileBasedAppArtifactsCommand(ParseResult parseResult) : CommandBase(parseResult)
+{
+    public override int Execute()
+    {
+        bool dryRun = _parseResult.GetValue(CleanFileBasedAppArtifactsCommandParser.DryRunOption);
+
+        using var metadataFileStream = OpenMetadataFile();
+
+        bool anyErrors = false;
+        int count = 0;
+
+        foreach (var folder in GetFoldersToRemove())
+        {
+            if (dryRun)
+            {
+                Reporter.Verbose.WriteLine($"Would remove folder: {folder.FullName}");
+                count++;
+            }
+            else
+            {
+                try
+                {
+                    folder.Delete(recursive: true);
+                    Reporter.Verbose.WriteLine($"Removed folder: {folder.FullName}");
+                    count++;
+                }
+                catch (Exception ex)
+                {
+                    Reporter.Error.WriteLine(string.Format(CliCommandStrings.CleanFileBasedAppArtifactsErrorRemovingFolder, folder, ex.Message).Red());
+                    anyErrors = true;
+                }
+            }
+        }
+
+        Reporter.Output.WriteLine(
+            dryRun
+                ? CliCommandStrings.CleanFileBasedAppArtifactsWouldRemoveFolders
+                : CliCommandStrings.CleanFileBasedAppArtifactsTotalFoldersRemoved,
+            count);
+
+        if (!dryRun)
+        {
+            UpdateMetadata(metadataFileStream);
+        }
+
+        return anyErrors ? 1 : 0;
+    }
+
+    private IEnumerable<DirectoryInfo> GetFoldersToRemove()
+    {
+        var directory = new DirectoryInfo(VirtualProjectBuildingCommand.GetTempSubdirectory());
+
+        if (!directory.Exists)
+        {
+            Reporter.Error.WriteLine(string.Format(CliCommandStrings.CleanFileBasedAppArtifactsDirectoryNotFound, directory.FullName).Yellow());
+            yield break;
+        }
+
+        Reporter.Output.WriteLine(CliCommandStrings.CleanFileBasedAppArtifactsScanning, directory.FullName);
+
+        var days = _parseResult.GetValue(CleanFileBasedAppArtifactsCommandParser.DaysOption);
+        var cutoff = DateTime.UtcNow.AddDays(-days);
+
+        foreach (var subdir in directory.GetDirectories())
+        {
+            if (subdir.LastWriteTimeUtc < cutoff)
+            {
+                yield return subdir;
+            }
+        }
+    }
+
+    private static FileInfo GetMetadataFile()
+    {
+        return new FileInfo(VirtualProjectBuildingCommand.GetTempSubpath(RunFileArtifactsMetadata.FilePath));
+    }
+
+    private FileStream? OpenMetadataFile()
+    {
+        if (!_parseResult.GetValue(CleanFileBasedAppArtifactsCommandParser.AutomaticOption))
+        {
+            return null;
+        }
+
+        // Open and lock the metadata file to ensure we are the only automatic cleanup process.
+        return GetMetadataFile().Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None);
+    }
+
+    private static void UpdateMetadata(FileStream? metadataFileStream)
+    {
+        if (metadataFileStream is null)
+        {
+            return;
+        }
+
+        var metadata = new RunFileArtifactsMetadata
+        {
+            LastAutomaticCleanupUtc = DateTime.UtcNow,
+        };
+        JsonSerializer.Serialize(metadataFileStream, metadata, RunFileJsonSerializerContext.Default.RunFileArtifactsMetadata);
+    }
+
+    /// <summary>
+    /// Starts a background process to clean up file-based app artifacts if needed.
+    /// </summary>
+    public static void StartAutomaticCleanupIfNeeded()
+    {
+        if (ShouldStartAutomaticCleanup())
+        {
+            Reporter.Verbose.WriteLine("Starting automatic cleanup of file-based app artifacts.");
+
+            var startInfo = new ProcessStartInfo
+            {
+                FileName = new Muxer().MuxerPath,
+                Arguments = ArgumentEscaper.EscapeAndConcatenateArgArrayForProcessStart(
+                [
+                    CleanCommandParser.GetCommand().Name,
+                    CleanFileBasedAppArtifactsCommandParser.Command.Name,
+                    CleanFileBasedAppArtifactsCommandParser.AutomaticOption.Name,
+                ]),
+                UseShellExecute = false,
+                RedirectStandardInput = true,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                CreateNoWindow = true,
+            };
+
+            Process.Start(startInfo);
+        }
+    }
+
+    private static bool ShouldStartAutomaticCleanup()
+    {
+        if (Env.GetEnvironmentVariableAsBool("DOTNET_CLI_DISABLE_FILE_BASED_APP_ARTIFACTS_AUTOMATIC_CLEANUP", defaultValue: false))
+        {
+            return false;
+        }
+
+        FileInfo? metadataFile = null;
+        try
+        {
+            metadataFile = GetMetadataFile();
+
+            if (!metadataFile.Exists)
+            {
+                return true;
+            }
+
+            using var stream = metadataFile.Open(FileMode.Open, FileAccess.Read, FileShare.Read);
+            var metadata = JsonSerializer.Deserialize(stream, RunFileJsonSerializerContext.Default.RunFileArtifactsMetadata);
+
+            if (metadata?.LastAutomaticCleanupUtc is not { } timestamp)
+            {
+                return true;
+            }
+
+            // Start automatic cleanup every two days.
+            return timestamp.AddDays(2) < DateTime.UtcNow;
+        }
+        catch (Exception ex)
+        {
+            Reporter.Verbose.WriteLine($"Cannot access artifacts metadata file '{metadataFile?.FullName}': {ex}");
+
+            // If the file cannot be accessed, automatic cleanup might already be running.
+            return false;
+        }
+    }
+}
+
+/// <summary>
+/// Metadata stored at the root level of the file-based app artifacts directory.
+/// </summary>
+internal sealed class RunFileArtifactsMetadata
+{
+    public const string FilePath = "dotnet-run-file-artifacts-metadata.json";
+
+    public DateTime? LastAutomaticCleanupUtc { get; init; }
+}

src/Cli/dotnet/Commands/Clean/FileBasedAppArtifacts/CleanFileBasedAppArtifactsCommandParser.cs

@@ -0,0 +1,49 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.CommandLine;
+
+namespace Microsoft.DotNet.Cli.Commands.Clean.FileBasedAppArtifacts;
+
+internal sealed class CleanFileBasedAppArtifactsCommandParser
+{
+    public static readonly Option<bool> DryRunOption = new("--dry-run")
+    {
+        Description = CliCommandStrings.CleanFileBasedAppArtifactsDryRun,
+        Arity = ArgumentArity.Zero,
+    };
+
+    public static readonly Option<int> DaysOption = new("--days")
+    {
+        Description = CliCommandStrings.CleanFileBasedAppArtifactsDays,
+        DefaultValueFactory = _ => 30,
+    };
+
+    /// <summary>
+    /// Specified internally when the command is started automatically in background by <c>dotnet run</c>.
+    /// Causes <see cref="RunFileArtifactsMetadata.LastAutomaticCleanupUtc"/> to be updated.
+    /// </summary>
+    public static readonly Option<bool> AutomaticOption = new("--automatic")
+    {
+        Hidden = true,
+    };
+
+    public static Command Command => field ??= ConstructCommand();
+
+    private static Command ConstructCommand()
+    {
+        Command command = new("file-based-apps", CliCommandStrings.CleanFileBasedAppArtifactsCommandDescription)
+        {
+            Hidden = true,
+            Options =
+            {
+                DryRunOption,
+                DaysOption,
+                AutomaticOption,
+            },
+        };
+
+        command.SetAction((parseResult) => new CleanFileBasedAppArtifactsCommand(parseResult).Execute());
+        return command;
+    }
+}

src/Cli/dotnet/Commands/CliCommandStrings.resx

@@ -274,6 +274,35 @@ Paths searched: '{1}', '{2}'.</value>
   <data name="CleanRuntimeOptionDescription" xml:space="preserve">
     <value>The target runtime to clean for.</value>
   </data>
+  <data name="CleanFileBasedAppArtifactsCommandDescription" xml:space="preserve">
+    <value>Removes artifacts created for file-based apps</value>
+  </data>
+  <data name="CleanFileBasedAppArtifactsDryRun" xml:space="preserve">
+    <value>Determines changes without actually modifying the file system</value>
+  </data>
+  <data name="CleanFileBasedAppArtifactsDays" xml:space="preserve">
+    <value>How many days an artifact folder needs to be unused in order to be removed</value>
+  </data>
+  <data name="CleanFileBasedAppArtifactsErrorRemovingFolder" xml:space="preserve">
+    <value>Error removing folder '{0}': {1}</value>
+    <comment>{0} is folder path. {1} is inner error message.</comment>
+  </data>
+  <data name="CleanFileBasedAppArtifactsWouldRemoveFolders" xml:space="preserve">
+    <value>Would remove folders: {0}</value>
+    <comment>{0} is count.</comment>
+  </data>
+  <data name="CleanFileBasedAppArtifactsTotalFoldersRemoved" xml:space="preserve">
+    <value>Total folders removed: {0}</value>
+    <comment>{0} is count.</comment>
+  </data>
+  <data name="CleanFileBasedAppArtifactsDirectoryNotFound" xml:space="preserve">
+    <value>Warning: Artifacts directory does not exist: {0}</value>
+    <comment>{0} is directory path.</comment>
+  </data>
+  <data name="CleanFileBasedAppArtifactsScanning" xml:space="preserve">
+    <value>Scanning for folders to remove in: {0}</value>
+    <comment>{0} is directory path.</comment>
+  </data>
   <data name="CmdBlameCrashCollectAlwaysDescription" xml:space="preserve">
     <value>Enables collecting crash dump on expected as well as unexpected testhost exit.</value>
   </data>

src/Cli/dotnet/Commands/Run/Api/RunApiCommand.cs

@@ -93,6 +93,7 @@ public override RunApiOutput Execute()
             {
                 CustomArtifactsPath = ArtifactsPath,
             };
+            buildCommand.MarkArtifactsFolderUsed();
 
             var runCommand = new RunCommand(
                 noBuild: false,

src/Cli/dotnet/Commands/Run/RunCommand.cs

@@ -140,7 +140,9 @@ public int Execute()
             if (EntryPointFileFullPath is not null)
             {
                 Debug.Assert(!ReadCodeFromStdin);
-                projectFactory = CreateVirtualCommand().CreateProjectInstance;
+                var command = CreateVirtualCommand();
+                command.MarkArtifactsFolderUsed();
+                projectFactory = command.CreateProjectInstance;
             }
         }
 
@@ -589,7 +591,7 @@ public static RunCommand FromParseResult(ParseResult parseResult)
             // If '-' is specified as the input file, read all text from stdin into a temporary file and use that as the entry point.
             // We create a new directory for each file so other files are not included in the compilation.
             // We fail if the file already exists to avoid reusing the same file for multiple stdin runs (in case the random name is duplicate).
-            string directory = VirtualProjectBuildingCommand.GetTempSubdirectory(Path.GetRandomFileName());
+            string directory = VirtualProjectBuildingCommand.GetTempSubpath(Path.GetRandomFileName());
             VirtualProjectBuildingCommand.CreateTempSubdirectory(directory);
             entryPointFilePath = Path.Join(directory, "app.cs");
             using (var stdinStream = Console.OpenStandardInput())