Rework the removal of unneeded file parts

ParadoxGameConverters · Nov 10, 2024 · ad43520 · ad43520
1 parent 5a84a9a
commit ad43520
Show file tree

Hide file tree

Showing 3 changed files with 59 additions and 19 deletions.
diff --git a/ImperatorToCK3/Outputter/FileTweaker.cs b/ImperatorToCK3/Outputter/FileTweaker.cs
@@ -1,42 +1,83 @@
 using commonItems;
+using commonItems.Collections;
 using commonItems.Mods;
 using ImperatorToCK3.CommonUtils;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
 
-namespace ImperatorToCK3.CK3.Cleanup;
+namespace ImperatorToCK3.Outputter;
+
+internal readonly struct PartOfFileToRemove(string text, bool warnIfNotFound = true) {
+	internal readonly string Text = text;
+	internal readonly bool WarnIfNotFound = warnIfNotFound;
+
+	public void Deconstruct(out string text, out bool warnIfNotFound) {
+		text = Text;
+		warnIfNotFound = WarnIfNotFound;
+	}
+}
 
 public static class FileTweaker {
 	public static async Task RemoveUnneededPartsOfFiles(ModFilesystem ck3ModFS, string outputModPath, Configuration config) {
+		// Load removable blocks from configurables.
+		Dictionary<string, OrderedSet<PartOfFileToRemove>> partsToRemovePerFile = new();
+
+		bool isVanilla = config.GetCK3ModFlags()["vanilla"];
+		Logger.Info("Reading unneeded parts of vanilla files...");
+		ReadPartsOfFileToRemove(partsToRemovePerFile, "configurables/removable_file_blocks.txt", warnIfNotFound: isVanilla);
+
 		if (config.FallenEagleEnabled) {
-			Logger.Info("Removing unneeded parts of Fallen Eagle files...");
-			await RemovePartsOfFilesFromConfigurable("configurables/removable_file_blocks_tfe.txt", ck3ModFS, outputModPath);
-		} else if (!config.WhenTheWorldStoppedMakingSenseEnabled) { // vanilla
-			Logger.Info("Removing unneeded parts of vanilla files...");
-			await RemovePartsOfFilesFromConfigurable("configurables/removable_file_blocks.txt", ck3ModFS, outputModPath);
+			Logger.Info("Reading unneeded parts of Fallen Eagle files...");
+			ReadPartsOfFileToRemove(partsToRemovePerFile, "configurables/removable_file_blocks_tfe.txt", warnIfNotFound: true);
 		}
+
+		await RemovePartsOfFiles(partsToRemovePerFile, ck3ModFS, outputModPath);
 	}
-
-	private static async Task RemovePartsOfFilesFromConfigurable(string configurablePath, ModFilesystem ck3ModFS, string outputModPath) {
-		// Load removable blocks from configurables.
-		Dictionary<string, string[]> partsToRemovePerFile = [];
+
+	private static void ReadPartsOfFileToRemove(Dictionary<string, OrderedSet<PartOfFileToRemove>> partsToRemovePerFile, string configurablePath, bool warnIfNotFound) {
 		var parser = new Parser();
 		parser.RegisterRegex(CommonRegexes.String, (reader, fileName) => {
 			var blocksToRemove = new BlobList(reader).Blobs.Select(b => b.Trim()).ToArray();
-			partsToRemovePerFile[fileName] = blocksToRemove;
+
+			if (partsToRemovePerFile.TryGetValue(fileName, out var existingBlocksToRemove)) {
+				var alreadyExistingBlocks = existingBlocksToRemove.Select(b => b.Text);
+				var newBlocksToRemove = blocksToRemove
+					.Except(alreadyExistingBlocks)
+					.Select(b => new PartOfFileToRemove(text: b, warnIfNotFound: warnIfNotFound));
+				existingBlocksToRemove.UnionWith(newBlocksToRemove);
+			} else {
+				partsToRemovePerFile[fileName] = blocksToRemove
+					.Select(b => new PartOfFileToRemove(text: b, warnIfNotFound: warnIfNotFound))
+					.ToOrderedSet();
+			}
 		});
 		parser.RegisterRegex(CommonRegexes.QuotedString, (reader, fileNameInQuotes) => {
 			var blocksToRemove = new BlobList(reader).Blobs.Select(b => b.Trim()).ToArray();
-			partsToRemovePerFile[fileNameInQuotes.RemQuotes()] = blocksToRemove;
+
+			var fileName = fileNameInQuotes.RemQuotes();
+			if (partsToRemovePerFile.TryGetValue(fileName, out var existingBlocksToRemove)) {
+				var alreadyExistingBlocks = existingBlocksToRemove.Select(b => b.Text);
+				var blocksToAdd = blocksToRemove
+					.Except(alreadyExistingBlocks)
+					.Select(b => new PartOfFileToRemove(text: b, warnIfNotFound: warnIfNotFound));
+				existingBlocksToRemove.UnionWith(blocksToAdd);
+			} else {
+				partsToRemovePerFile[fileName] = blocksToRemove
+					.Select(b => new PartOfFileToRemove(text: b, warnIfNotFound: warnIfNotFound))
+					.ToOrderedSet();
+			}
 		});
 		parser.IgnoreAndLogUnregisteredItems();
-		parser.ParseFile(configurablePath);
 
+		parser.ParseFile(configurablePath);
+	}
+
+	private static async Task RemovePartsOfFiles(Dictionary<string, OrderedSet<PartOfFileToRemove>> partsToRemovePerFile, ModFilesystem ck3ModFS, string outputModPath) {
 		// Log count of blocks to remove for each file.
 		foreach (var (relativePath, partsToRemove) in partsToRemovePerFile) {
-			Logger.Debug($"Loaded {partsToRemove.Length} blocks to remove from {relativePath}.");
+			Logger.Debug($"Loaded {partsToRemove.Count} blocks to remove from {relativePath}.");
 		}
 
 		foreach (var (relativePath, partsToRemove) in partsToRemovePerFile) {
@@ -50,7 +91,7 @@ private static async Task RemovePartsOfFilesFromConfigurable(string configurable
 
 			var fileContent = await File.ReadAllTextAsync(inputPath);
 
-			foreach (var block in partsToRemove) {
+			foreach (var (block, warnIfNotFound) in partsToRemove) {
 				// If the file uses other line endings than CRLF, we need to modify the search string.
 				string searchString;
 				if (lineEndings == "LF") {
@@ -61,9 +102,10 @@ private static async Task RemovePartsOfFilesFromConfigurable(string configurable
 					searchString = block;
 				}
 
-				// Log if the block is not found.
 				if (!fileContent.Contains(searchString)) {
-					Logger.Warn($"Block not found in file {relativePath}: {searchString}");
+					if (warnIfNotFound) {
+						Logger.Warn($"Block not found in file {relativePath}: {searchString}");
+					}
 					continue;
 				}
 

diff --git a/ImperatorToCK3/Outputter/OnActionOutputter.cs b/ImperatorToCK3/Outputter/OnActionOutputter.cs
@@ -1,7 +1,6 @@
 using commonItems;
 using commonItems.Collections;
 using commonItems.Mods;
-using ImperatorToCK3.CK3.Cleanup;
 using ImperatorToCK3.CommonUtils;
 using System.Collections.Generic;
 using System.IO;

diff --git a/ImperatorToCK3/Outputter/WorldOutputter.cs b/ImperatorToCK3/Outputter/WorldOutputter.cs
@@ -4,7 +4,6 @@
 using commonItems.Serialization;
 using DotLiquid;
 using ImperatorToCK3.CK3;
-using ImperatorToCK3.CK3.Cleanup;
 using ImperatorToCK3.CK3.Legends;
 using ImperatorToCK3.CommonUtils;
 using ImperatorToCK3.Exceptions;