From 9c27090a3baa7ca38b0a272552a7a7708a677f95 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 1 Jul 2024 20:24:48 -0400 Subject: [PATCH 001/176] Add build.zig.zon, update build.zig, .gitignore This adds a test running step and a zig.zon manifest. Also adds to the .gitignore the new location of caches, .zig-out. --- .gitignore | 3 ++- build.zig | 26 ++++++++++++++++++++++++++ build.zig.zon | 11 +++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 build.zig.zon diff --git a/.gitignore b/.gitignore index 2040c29..68557b5 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -zig-cache +zig-* +.zig-* diff --git a/build.zig b/build.zig index 30802da..d144529 100644 --- a/build.zig +++ b/build.zig @@ -1,7 +1,33 @@ const std = @import("std"); pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + _ = b.addModule("diffz", .{ .root_source_file = b.path("DiffMatchPatch.zig"), }); + + const lib = b.addStaticLibrary(.{ + .name = "diffz", + .root_source_file = b.path("DiffMatchPatch.zig"), + .target = target, + .optimize = optimize, + }); + + // This declares intent for the library to be installed into the standard + // location when the user invokes the "install" step (the default step when + // running `zig build`). + b.installArtifact(lib); + + // Run tests + const tests = b.addTest(.{ + .name = "tests", + .root_source_file = b.path("DiffMatchPatch.zig"), + .target = target, + .optimize = optimize, + }); + const step_tests = b.addRunArtifact(tests); + + b.step("test", "Run diffz tests").dependOn(&step_tests.step); } diff --git a/build.zig.zon b/build.zig.zon new file mode 100644 index 0000000..747b11a --- /dev/null +++ b/build.zig.zon @@ -0,0 +1,11 @@ +.{ + .name = "DiffMatchPatch", + .version = "0.0.1", + .paths = .{ + "DiffMatchPatch.zig", + "LICENSE", + "README.md", + "build.zig.zon", + "build.zig", + }, +} From 4792861d6d423f6a8fead9db6c7f0b0024b22136 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 1 Jul 2024 20:30:34 -0400 Subject: [PATCH 002/176] Module name is diffz, not DiffMatchPatch --- build.zig.zon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.zig.zon b/build.zig.zon index 747b11a..48ea526 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,5 +1,5 @@ .{ - .name = "DiffMatchPatch", + .name = "diffz", .version = "0.0.1", .paths = .{ "DiffMatchPatch.zig", From 10bcba04b69861e6badffed87bc09462369c1ee5 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 1 Jul 2024 20:49:26 -0400 Subject: [PATCH 003/176] More paths, .target, .optimize --- build.zig | 2 ++ build.zig.zon | 2 ++ 2 files changed, 4 insertions(+) diff --git a/build.zig b/build.zig index d144529..dd40eb6 100644 --- a/build.zig +++ b/build.zig @@ -6,6 +6,8 @@ pub fn build(b: *std.Build) void { _ = b.addModule("diffz", .{ .root_source_file = b.path("DiffMatchPatch.zig"), + .target = target, + .optimize = optimize, }); const lib = b.addStaticLibrary(.{ diff --git a/build.zig.zon b/build.zig.zon index 48ea526..b4e0b4d 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -3,6 +3,8 @@ .version = "0.0.1", .paths = .{ "DiffMatchPatch.zig", + ".gitattributes", + ".gitignore", "LICENSE", "README.md", "build.zig.zon", From 64bc1bf730a8be35b53d30ce593042b59432cb98 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 11:39:35 -0400 Subject: [PATCH 004/176] Memory-managed halfMatch function halfMatch now correctly manages its own memory, like it's supposed to. --- DiffMatchPatch.zig | 134 +++++++++++++++++++++++++++------------------ 1 file changed, 82 insertions(+), 52 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 3540518..c18b2e0 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2,6 +2,7 @@ const DiffMatchPatch = @This(); const std = @import("std"); const testing = std.testing; +const Allocator = std.mem.Allocator; const ArrayListUnmanaged = std.ArrayListUnmanaged; const DiffList = ArrayListUnmanaged(Diff); @@ -255,6 +256,16 @@ const HalfMatchResult = struct { prefix_after: []const u8, suffix_after: []const u8, common_middle: []const u8, + + // TODO maybe check for empty slice here for fewer copies, + // as in, maybe we can transfer ownership and replace with "". + pub fn deinit(hmr: HalfMatchResult, alloc: Allocator) void { + alloc.free(hmr.prefix_before); + alloc.free(hmr.suffix_before); + alloc.free(hmr.prefix_after); + alloc.free(hmr.suffix_after); + alloc.free(hmr.common_middle); + } }; /// Do the two texts share a Substring which is at least half the length of @@ -296,16 +307,22 @@ fn diffHalfMatch( half_match = half_match_2.?; } else { // Both matched. Select the longest. - half_match = if (half_match_1.?.common_middle.len > half_match_2.?.common_middle.len) - half_match_1 - else - half_match_2; + half_match = half: { + if (half_match_1.?.common_middle.len > half_match_2.?.common_middle.len) { + half_match_2.?.deinit(allocator); + break :half half_match_1; + } else { + half_match_1.?.deinit(allocator); + break :half half_match_2; + } + }; } // A half-match was found, sort out the return data. if (before.len > after.len) { - return half_match; + return half_match.?; } else { + // Transfers ownership of all memory to new, permuted, half_match. const half_match_yes = half_match.?; return .{ .prefix_before = half_match_yes.prefix_after, @@ -337,6 +354,7 @@ fn diffHalfMatchInternal( var j: isize = -1; var best_common = std.ArrayListUnmanaged(u8){}; + defer best_common.deinit(allocator); var best_long_text_a: []const u8 = ""; var best_long_text_b: []const u8 = ""; var best_short_text_a: []const u8 = ""; @@ -350,8 +368,10 @@ fn diffHalfMatchInternal( const suffix_length = diffCommonSuffix(long_text[0..i], short_text[0..@as(usize, @intCast(j))]); if (best_common.items.len < suffix_length + prefix_length) { best_common.items.len = 0; - try best_common.appendSlice(allocator, short_text[@as(usize, @intCast(j - @as(isize, @intCast(suffix_length)))) .. @as(usize, @intCast(j - @as(isize, @intCast(suffix_length)))) + suffix_length]); - try best_common.appendSlice(allocator, short_text[@as(usize, @intCast(j)) .. @as(usize, @intCast(j)) + prefix_length]); + const a = short_text[@as(usize, @intCast(j - @as(isize, @intCast(suffix_length)))) .. @as(usize, @intCast(j - @as(isize, @intCast(suffix_length)))) + suffix_length]; + try best_common.appendSlice(allocator, a); + const b = short_text[@as(usize, @intCast(j)) .. @as(usize, @intCast(j)) + prefix_length]; + try best_common.appendSlice(allocator, b); best_long_text_a = long_text[0 .. i - suffix_length]; best_long_text_b = long_text[i + prefix_length ..]; @@ -361,11 +381,11 @@ fn diffHalfMatchInternal( } if (best_common.items.len * 2 >= long_text.len) { return .{ - .prefix_before = best_long_text_a, - .suffix_before = best_long_text_b, - .prefix_after = best_short_text_a, - .suffix_after = best_short_text_b, - .common_middle = best_common.items, + .prefix_before = try allocator.dupe(u8, best_long_text_a), + .suffix_before = try allocator.dupe(u8, best_long_text_b), + .prefix_after = try allocator.dupe(u8, best_short_text_a), + .suffix_after = try allocator.dupe(u8, best_short_text_b), + .common_middle = try best_common.toOwnedSlice(allocator), }; } else { return null; @@ -1411,97 +1431,107 @@ test diffCommonOverlap { } test diffHalfMatch { - var arena = std.heap.ArenaAllocator.init(testing.allocator); - defer arena.deinit(); + const allocator = testing.allocator; var one_timeout = DiffMatchPatch{}; one_timeout.diff_timeout = 1; - + const dh1 = try one_timeout.diffHalfMatch(allocator, "1234567890", "abcdef"); try testing.expectEqual( @as(?HalfMatchResult, null), - try one_timeout.diffHalfMatch(arena.allocator(), "1234567890", "abcdef"), + dh1, ); // No match #1 + const dh2 = try one_timeout.diffHalfMatch(allocator, "12345", "23"); try testing.expectEqual( @as(?HalfMatchResult, null), - try one_timeout.diffHalfMatch(arena.allocator(), "12345", "23"), + dh2, ); // No match #2 // Single matches - try testing.expectEqualDeep(@as(?HalfMatchResult, HalfMatchResult{ + var dh3 = (try one_timeout.diffHalfMatch(allocator, "1234567890", "a345678z")).?; + defer dh3.deinit(allocator); + try testing.expectEqualDeep(HalfMatchResult{ .prefix_before = "12", .suffix_before = "90", .prefix_after = "a", .suffix_after = "z", .common_middle = "345678", - }), try one_timeout.diffHalfMatch(arena.allocator(), "1234567890", "a345678z")); // Single Match #1 + }, dh3); // Single Match #1 - try testing.expectEqualDeep(@as(?HalfMatchResult, HalfMatchResult{ + var dh4 = (try one_timeout.diffHalfMatch(allocator, "a345678z", "1234567890")).?; + defer dh4.deinit(allocator); + try testing.expectEqualDeep(HalfMatchResult{ .prefix_before = "a", .suffix_before = "z", .prefix_after = "12", .suffix_after = "90", .common_middle = "345678", - }), try one_timeout.diffHalfMatch(arena.allocator(), "a345678z", "1234567890")); // Single Match #2 + }, dh4); // Single Match #2 - try testing.expectEqualDeep(@as(?HalfMatchResult, HalfMatchResult{ + var dh5 = (try one_timeout.diffHalfMatch(allocator, "abc56789z", "1234567890")).?; + defer dh5.deinit(allocator); + try testing.expectEqualDeep(HalfMatchResult{ .prefix_before = "abc", .suffix_before = "z", .prefix_after = "1234", .suffix_after = "0", .common_middle = "56789", - }), try one_timeout.diffHalfMatch(arena.allocator(), "abc56789z", "1234567890")); // Single Match #3 + }, dh5); // Single Match #3 - try testing.expectEqualDeep(@as(?HalfMatchResult, HalfMatchResult{ + var dh6 = (try one_timeout.diffHalfMatch(allocator, "a23456xyz", "1234567890")).?; + defer dh6.deinit(allocator); + try testing.expectEqualDeep(HalfMatchResult{ .prefix_before = "a", .suffix_before = "xyz", .prefix_after = "1", .suffix_after = "7890", .common_middle = "23456", - }), try one_timeout.diffHalfMatch(arena.allocator(), "a23456xyz", "1234567890")); // Single Match #4 + }, dh6); // Single Match #4 // Multiple matches - try testing.expectEqualDeep( - @as(?HalfMatchResult, HalfMatchResult{ - .prefix_before = "12123", - .suffix_before = "123121", - .prefix_after = "a", - .suffix_after = "z", - .common_middle = "1234123451234", - }), - try one_timeout.diffHalfMatch(arena.allocator(), "121231234123451234123121", "a1234123451234z"), - ); // Multiple Matches #1 - - try testing.expectEqualDeep( - @as(?HalfMatchResult, HalfMatchResult{ - .prefix_before = "", - .suffix_before = "-=-=-=-=-=", - .prefix_after = "x", - .suffix_after = "", - .common_middle = "x-=-=-=-=-=-=-=", - }), - try one_timeout.diffHalfMatch(arena.allocator(), "x-=-=-=-=-=-=-=-=-=-=-=-=", "xx-=-=-=-=-=-=-="), - ); // Multiple Matches #2 - - try testing.expectEqualDeep(@as(?HalfMatchResult, HalfMatchResult{ + var dh7 = (try one_timeout.diffHalfMatch(allocator, "121231234123451234123121", "a1234123451234z")).?; + defer dh7.deinit(allocator); + try testing.expectEqualDeep(HalfMatchResult{ + .prefix_before = "12123", + .suffix_before = "123121", + .prefix_after = "a", + .suffix_after = "z", + .common_middle = "1234123451234", + }, dh7); // Multiple Matches #1 + + var dh8 = (try one_timeout.diffHalfMatch(allocator, "x-=-=-=-=-=-=-=-=-=-=-=-=", "xx-=-=-=-=-=-=-=")).?; + defer dh8.deinit(allocator); + try testing.expectEqualDeep(HalfMatchResult{ + .prefix_before = "", + .suffix_before = "-=-=-=-=-=", + .prefix_after = "x", + .suffix_after = "", + .common_middle = "x-=-=-=-=-=-=-=", + }, dh8); // Multiple Matches #2 + + var dh9 = (try one_timeout.diffHalfMatch(allocator, "-=-=-=-=-=-=-=-=-=-=-=-=y", "-=-=-=-=-=-=-=yy")).?; + defer dh9.deinit(allocator); + try testing.expectEqualDeep(HalfMatchResult{ .prefix_before = "-=-=-=-=-=", .suffix_before = "", .prefix_after = "", .suffix_after = "y", .common_middle = "-=-=-=-=-=-=-=y", - }), try one_timeout.diffHalfMatch(arena.allocator(), "-=-=-=-=-=-=-=-=-=-=-=-=y", "-=-=-=-=-=-=-=yy")); // Multiple Matches #3 + }, dh9); // Multiple Matches #3 // Other cases // Optimal diff would be -q+x=H-i+e=lloHe+Hu=llo-Hew+y not -qHillo+x=HelloHe-w+Hulloy - try testing.expectEqualDeep(@as(?HalfMatchResult, HalfMatchResult{ + var dh10 = (try one_timeout.diffHalfMatch(allocator, "qHilloHelloHew", "xHelloHeHulloy")).?; + defer dh10.deinit(allocator); + try testing.expectEqualDeep(HalfMatchResult{ .prefix_before = "qHillo", .suffix_before = "w", .prefix_after = "x", .suffix_after = "Hulloy", .common_middle = "HelloHe", - }), try one_timeout.diffHalfMatch(arena.allocator(), "qHilloHelloHew", "xHelloHeHulloy")); // Non-optimal halfmatch + }, dh10); // Non-optimal halfmatch one_timeout.diff_timeout = 0; - try testing.expectEqualDeep(@as(?HalfMatchResult, null), try one_timeout.diffHalfMatch(arena.allocator(), "qHilloHelloHew", "xHelloHeHulloy")); // Non-optimal halfmatch + try testing.expectEqualDeep(@as(?HalfMatchResult, null), try one_timeout.diffHalfMatch(allocator, "qHilloHelloHew", "xHelloHeHulloy")); // Non-optimal halfmatch } test diffLinesToChars { From 0c7c4e29dd840f810a311384442a5602c06ca7a2 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 12:14:18 -0400 Subject: [PATCH 005/176] Managed memory in diffLinesToChars --- DiffMatchPatch.zig | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index c18b2e0..4131a4e 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -413,8 +413,10 @@ fn diffBisect( const v_length = 2 * max_d; var v1 = try ArrayListUnmanaged(isize).initCapacity(allocator, @as(usize, @intCast(v_length))); + defer v1.deinit(allocator); v1.items.len = @intCast(v_length); var v2 = try ArrayListUnmanaged(isize).initCapacity(allocator, @as(usize, @intCast(v_length))); + defer v2.deinit(allocator); v2.items.len = @intCast(v_length); var x: usize = 0; @@ -650,6 +652,12 @@ const LinesToCharsResult = struct { chars_1: []const u8, chars_2: []const u8, line_array: ArrayListUnmanaged([]const u8), + + pub fn deinit(self: *LinesToCharsResult, allocator: Allocator) void { + allocator.free(self.chars_1); + allocator.free(self.chars_2); + self.line_array.deinit(allocator); + } }; /// Split two texts into a list of strings. Reduce the texts to a string of @@ -665,12 +673,15 @@ fn diffLinesToChars( text2: []const u8, ) DiffError!LinesToCharsResult { var line_array = ArrayListUnmanaged([]const u8){}; + errdefer line_array.deinit(allocator); var line_hash = std.StringHashMapUnmanaged(usize){}; + defer line_hash.deinit(allocator); // e.g. line_array[4] == "Hello\n" // e.g. line_hash.get("Hello\n") == 4 // "\x00" is a valid character, but various debuggers don't like it. // So we'll insert a junk entry to avoid generating a null character. + // XXX why is this necessary? -Sam try line_array.append(allocator, ""); // Allocate 2/3rds of the space for text1, the rest for text2. @@ -697,9 +708,9 @@ fn diffLinesToCharsMunge( var line_end: isize = -1; var line: []const u8 = ""; var chars = ArrayListUnmanaged(u8){}; + defer chars.deinit(allocator); // Walk the text, pulling out a Substring for each line. - // text.split('\n') would would temporarily double our memory footprint. - // Modifying text would create many large strings to garbage collect. + // TODO this can be handled with a Reader, avoiding all the manual splitting while (line_end < @as(isize, @intCast(text.len)) - 1) { line_end = b: { break :b @as(isize, @intCast(std.mem.indexOf(u8, text[@intCast(line_start)..], "\n") orelse @@ -1535,16 +1546,15 @@ test diffHalfMatch { } test diffLinesToChars { - var arena = std.heap.ArenaAllocator.init(testing.allocator); - defer arena.deinit(); - + const allocator = std.testing.allocator; // Convert lines down to characters. - var tmp_array_list = std.ArrayList([]const u8).init(arena.allocator()); + var tmp_array_list = std.ArrayList([]const u8).init(allocator); + defer tmp_array_list.deinit(); try tmp_array_list.append(""); try tmp_array_list.append("alpha\n"); try tmp_array_list.append("beta\n"); - var result = try diffLinesToChars(arena.allocator(), "alpha\nbeta\nalpha\n", "beta\nalpha\nbeta\n"); + var result = try diffLinesToChars(allocator, "alpha\nbeta\nalpha\n", "beta\nalpha\nbeta\n"); try testing.expectEqualStrings("\u{0001}\u{0002}\u{0001}", result.chars_1); // Shared lines #1 try testing.expectEqualStrings("\u{0002}\u{0001}\u{0002}", result.chars_2); // Shared lines #2 try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // Shared lines #3 @@ -1554,8 +1564,9 @@ test diffLinesToChars { try tmp_array_list.append("alpha\r\n"); try tmp_array_list.append("beta\r\n"); try tmp_array_list.append("\r\n"); + result.deinit(allocator); - result = try diffLinesToChars(arena.allocator(), "", "alpha\r\nbeta\r\n\r\n\r\n"); + result = try diffLinesToChars(allocator, "", "alpha\r\nbeta\r\n\r\n\r\n"); try testing.expectEqualStrings("", result.chars_1); // Empty string and blank lines #1 try testing.expectEqualStrings("\u{0001}\u{0002}\u{0003}\u{0003}", result.chars_2); // Empty string and blank lines #2 try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // Empty string and blank lines #3 @@ -1565,11 +1576,13 @@ test diffLinesToChars { try tmp_array_list.append("a"); try tmp_array_list.append("b"); - result = try diffLinesToChars(arena.allocator(), "a", "b"); + result.deinit(allocator); + result = try diffLinesToChars(allocator, "a", "b"); try testing.expectEqualStrings("\u{0001}", result.chars_1); // No linebreaks #1. try testing.expectEqualStrings("\u{0002}", result.chars_2); // No linebreaks #2. try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // No linebreaks #3. + result.deinit(allocator); // TODO: More than 256 to reveal any 8-bit limitations but this requires // some unicode logic that I don't want to deal with @@ -1578,8 +1591,8 @@ test diffLinesToChars { // const n: u8 = 255; // tmp_array_list.items.len = 0; - // var line_list = std.ArrayList(u8).init(arena.allocator()); - // var char_list = std.ArrayList(u8).init(arena.allocator()); + // var line_list = std.ArrayList(u8).init(alloc); + // var char_list = std.ArrayList(u8).init(alloc); // var i: u8 = 0; // while (i < n) : (i += 1) { @@ -1590,7 +1603,7 @@ test diffLinesToChars { // try testing.expectEqual(@as(usize, n), tmp_array_list.items.len); // Test initialization fail #1 // try testing.expectEqual(@as(usize, n), char_list.items.len); // Test initialization fail #2 // try tmp_array_list.insert(0, ""); - // result = try diffLinesToChars(arena.allocator(), line_list.items, ""); + // result = try diffLinesToChars(alloc, line_list.items, ""); // try testing.expectEqualStrings(char_list.items, result.chars_1); // try testing.expectEqualStrings("", result.chars_2); // try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); From 9e638e8e90885e20597baa4c8c1fa383c60b41a7 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 15:14:34 -0400 Subject: [PATCH 006/176] Clean up easy leaks + prep for allocating Diffs The tests currently create Diffs with constant/static text, which causes a panic on any attempt to free said text. This pass fixes a couple leaks, and lays a foundation for consistently freeing any diff's .text field when no longer in use: this requires that, as it will be in real programs, the diff text is allocated to begin with. --- DiffMatchPatch.zig | 98 +++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 4131a4e..6458df4 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -6,6 +6,28 @@ const Allocator = std.mem.Allocator; const ArrayListUnmanaged = std.ArrayListUnmanaged; const DiffList = ArrayListUnmanaged(Diff); +fn deinitDiffList(allocator: Allocator, diffs: *DiffList) void { + defer diffs.deinit(allocator); + for (diffs.items) |d| { + if (d.text.len > 0) { + allocator.free(d.text); + } + } +} + +fn freeRangeDiffList( + allocator: Allocator, + diffs: *DiffList, + start: usize, + len: usize, +) void { + const after_range = start + len; + const range = diffs.items[start..after_range]; + for (range) |d| { + allocator.free(d.text); + } +} + /// DMP with default configuration options pub const default = DiffMatchPatch{}; @@ -207,15 +229,15 @@ fn diffCompute( if (short_text.len == 1) { // Single character string. // After the previous speedup, the character can't be an equality. - try diffs.append(allocator, Diff.init(.delete, before)); - try diffs.append(allocator, Diff.init(.insert, after)); + try diffs.append(allocator, Diff.init(.delete, try allocator.dupe(u8, before))); + try diffs.append(allocator, Diff.init(.insert, try allocator.dupe(u8, after))); return diffs; } // Check to see if the problem can be split in two. if (try dmp.diffHalfMatch(allocator, before, after)) |half_match| { // A half-match was found, sort out the return data. - + defer half_match.deinit(allocator); // Send both pairs off for separate processing. const diffs_a = try dmp.diffInternal( allocator, @@ -238,7 +260,7 @@ fn diffCompute( // Merge the results. diffs = diffs_a; - try diffs.append(allocator, Diff.init(.equal, half_match.common_middle)); + try diffs.append(allocator, Diff.init(.equal, try allocator.dupe(u8, half_match.common_middle))); try diffs.appendSlice(allocator, diffs_b.items); return diffs; } @@ -579,7 +601,8 @@ fn diffLineMode( deadline: u64, ) DiffError!DiffList { // Scan the text on a line-by-line basis first. - const a = try diffLinesToChars(allocator, text1_in, text2_in); + var a = try diffLinesToChars(allocator, text1_in, text2_in); + defer a.deinit(allocator); const text1 = a.chars_1; const text2 = a.chars_2; const line_array = a.line_array; @@ -796,22 +819,14 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo if ((pointer - count_delete - count_insert) > 0 and diffs.items[pointer - count_delete - count_insert - 1].operation == .equal) { - // diffs.items[pointer - count_delete - count_insert - 1].text - // += text_insert.Substring(0, common_length); - const ii = pointer - count_delete - count_insert - 1; var nt = try allocator.alloc(u8, diffs.items[ii].text.len + common_length); - // try diffs.items[pointer - count_delete - count_insert - 1].text.append(allocator, text_insert.items[0..common_length]); const ot = diffs.items[ii].text; @memcpy(nt[0..ot.len], ot); @memcpy(nt[ot.len..], text_insert.items[0..common_length]); - - // allocator.free(diffs.items[ii].text); diffs.items[ii].text = nt; } else { - // diffs.Insert(0, Diff.init(.equal, - // text_insert.Substring(0, common_length))); const text = std.ArrayListUnmanaged(u8){ .items = try allocator.dupe(u8, text_insert.items[0..common_length]), }; @@ -853,19 +868,11 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo } else if (pointer != 0 and diffs.items[pointer - 1].operation == .equal) { // Merge this equality with the previous one. // TODO: Fix using realloc or smth - var nt = try allocator.alloc(u8, diffs.items[pointer - 1].text.len + diffs.items[pointer].text.len); - - // try diffs.items[pointer - count_delete - count_insert - 1].text.append(allocator, text_insert.items[0..common_length]); const ot = diffs.items[pointer - 1].text; @memcpy(nt[0..ot.len], ot); @memcpy(nt[ot.len..], diffs.items[pointer].text); - - // allocator.free(diffs.items[pointer - 1].text); diffs.items[pointer - 1].text = nt; - // allocator.free(diffs.items[pointer].text); - - // try diffs.items[pointer - 1].text.append(allocator, diffs.items[pointer].text.items); _ = diffs.orderedRemove(pointer); } else { pointer += 1; @@ -893,12 +900,6 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo { // This is a single edit surrounded by equalities. if (std.mem.endsWith(u8, diffs.items[pointer].text, diffs.items[pointer - 1].text)) { - // Shift the edit over the previous equality. - // diffs.items[pointer].text = diffs.items[pointer - 1].text + - // diffs.items[pointer].text[0 .. diffs.items[pointer].text.len - - // diffs.items[pointer - 1].text.len]; - // diffs.items[pointer + 1].text = diffs.items[pointer - 1].text + diffs.items[pointer + 1].text; - const pt = try std.mem.concat(allocator, u8, &.{ diffs.items[pointer - 1].text, diffs.items[pointer].text[0 .. diffs.items[pointer].text.len - @@ -908,21 +909,12 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo diffs.items[pointer - 1].text, diffs.items[pointer + 1].text, }); - - // allocator.free(diffs.items[pointer].text); - // allocator.free(diffs.items[pointer + 1].text); - diffs.items[pointer].text = pt; diffs.items[pointer + 1].text = p1t; - + // XXX reactivate freeRangeDiffList(allocator, diffs, pointer - 1, 1); try diffs.replaceRange(allocator, pointer - 1, 1, &.{}); changes = true; } else if (std.mem.startsWith(u8, diffs.items[pointer].text, diffs.items[pointer + 1].text)) { - // Shift the edit over the next equality. - // diffs.items[pointer - 1].text += diffs.items[pointer + 1].text; - // diffs.items[pointer].text = - // diffs.items[pointer].text[diffs.items[pointer + 1].text.len..] + diffs.items[pointer + 1].text; - const pm1t = try std.mem.concat(allocator, u8, &.{ diffs.items[pointer - 1].text, diffs.items[pointer + 1].text, @@ -931,13 +923,9 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo diffs.items[pointer].text[diffs.items[pointer + 1].text.len..], diffs.items[pointer + 1].text, }); - - // allocator.free(diffs.items[pointer - 1].text); - // allocator.free(diffs.items[pointer].text); - diffs.items[pointer - 1].text = pm1t; diffs.items[pointer].text = pt; - + // XXX reactivate freeRangeDiffList(allocator, diffs, pointer - 1, 1); try diffs.replaceRange(allocator, pointer + 1, 1, &.{}); changes = true; } @@ -1043,8 +1031,10 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError @intCast(pointer), Diff.init(.equal, try allocator.dupe(u8, insertion[0..overlap_length1])), ); + // XXX activate: allocator.free(diffs.items[@inteCast(pointer-1)].text); diffs.items[@intCast(pointer - 1)].text = try allocator.dupe(u8, deletion[0 .. deletion.len - overlap_length1]); + // XXX activate: allocator.free(diffs.items[@inteCast(pointer+1)].text); diffs.items[@intCast(pointer + 1)].text = try allocator.dupe(u8, insertion[overlap_length1..]); pointer += 1; @@ -1061,11 +1051,13 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError Diff.init(.equal, try allocator.dupe(u8, deletion[0..overlap_length2])), ); diffs.items[@intCast(pointer - 1)].operation = .insert; - diffs.items[@intCast(pointer - 1)].text = - try allocator.dupe(u8, insertion[0 .. insertion.len - overlap_length2]); + const new_minus = try allocator.dupe(u8, insertion[0 .. insertion.len - overlap_length2]); + // XXX activate: allocator.free(diffs.items[@inteCast(pointer-1)].text); + diffs.items[@intCast(pointer - 1)].text = new_minus; diffs.items[@intCast(pointer + 1)].operation = .delete; - diffs.items[@intCast(pointer + 1)].text = - try allocator.dupe(u8, deletion[overlap_length2..]); + const new_plus = try allocator.dupe(u8, deletion[overlap_length2..]); + // XXX activate: allocator.free(diffs.items[@inteCast(pointer+1)].text); + diffs.items[@intCast(pointer + 1)].text = new_plus; pointer += 1; } } @@ -1114,7 +1106,7 @@ pub fn diffCleanupSemanticLossless( const not_common = try allocator.dupe(u8, edit.items[0 .. edit.items.len - common_offset]); defer allocator.free(not_common); - edit.items.len = 0; + edit.clearRetainingCapacity(); try edit.appendSlice(allocator, common_string); try edit.appendSlice(allocator, not_common); @@ -1167,16 +1159,23 @@ pub fn diffCleanupSemanticLossless( if (!std.mem.eql(u8, diffs.items[pointer - 1].text, best_equality_1.items)) { // We have an improvement, save it back to the diff. if (best_equality_1.items.len != 0) { + // allocator.free(diffs.items[pointer - 1].text); diffs.items[pointer - 1].text = try allocator.dupe(u8, best_equality_1.items); } else { - _ = diffs.orderedRemove(pointer - 1); + const old_diff = diffs.orderedRemove(pointer - 1); + // allocator.free(old_diff.text); + _ = old_diff; pointer -= 1; } + // allocator.free(diffs.items[pointer].text); diffs.items[pointer].text = try allocator.dupe(u8, best_edit.items); if (best_equality_2.items.len != 0) { + // allocator.free(diffs.items[pointer - 1].text); diffs.items[pointer + 1].text = try allocator.dupe(u8, best_equality_2.items); } else { - _ = diffs.orderedRemove(pointer + 1); + const old_diff = diffs.orderedRemove(pointer + 1); + // allocator.free(old_diff.text); + _ = old_diff; pointer -= 1; } } @@ -1259,6 +1258,7 @@ pub fn diffCleanupEfficiency( var changes = false; // Stack of indices where equalities are found. var equalities = DiffList{}; + defer deinitDiffList(allocator, equalities); // Always equal to equalities[equalitiesLength-1][1] var last_equality = ""; var pointer: isize = 0; // Index of current position. From a5c993f07ac792c51f1744eaae3b478a68754f18 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 16:08:36 -0400 Subject: [PATCH 007/176] Managed memory in diffCharsToLines --- DiffMatchPatch.zig | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 6458df4..c34ea65 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -776,6 +776,7 @@ fn diffCharsToLines( while (j < d.text.len) : (j += 1) { try text.appendSlice(allocator, line_array[d.text[j]]); } + allocator.free(d.text); d.text = try allocator.dupe(u8, text.items); } } @@ -1575,14 +1576,14 @@ test diffLinesToChars { try tmp_array_list.append(""); try tmp_array_list.append("a"); try tmp_array_list.append("b"); - result.deinit(allocator); + result = try diffLinesToChars(allocator, "a", "b"); try testing.expectEqualStrings("\u{0001}", result.chars_1); // No linebreaks #1. try testing.expectEqualStrings("\u{0002}", result.chars_2); // No linebreaks #2. try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // No linebreaks #3. - result.deinit(allocator); + // TODO: More than 256 to reveal any 8-bit limitations but this requires // some unicode logic that I don't want to deal with @@ -1612,23 +1613,33 @@ test diffLinesToChars { test diffCharsToLines { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); - - try testing.expect((Diff.init(.equal, "a")).eql(Diff.init(.equal, "a"))); - try testing.expect(!(Diff.init(.insert, "a")).eql(Diff.init(.equal, "a"))); - try testing.expect(!(Diff.init(.equal, "a")).eql(Diff.init(.equal, "b"))); - try testing.expect(!(Diff.init(.equal, "a")).eql(Diff.init(.delete, "b"))); + const alloc = std.testing.allocator; + const equal_a = Diff.init(.equal, try alloc.dupe(u8, "a")); + defer alloc.free(equal_a.text); + const insert_a = Diff.init(.insert, try alloc.dupe(u8, "a")); + defer alloc.free(insert_a.text); + const equal_b = Diff.init(.equal, try alloc.dupe(u8, "b")); + defer alloc.free(equal_b.text); + const delete_b = Diff.init(.delete, try alloc.dupe(u8, "b")); + defer alloc.free(delete_b.text); + try testing.expect(equal_a.eql(equal_a)); + try testing.expect(!insert_a.eql(equal_a)); + try testing.expect(!equal_a.eql(equal_b)); + try testing.expect(!equal_a.eql(delete_b)); // Convert chars up to lines. - var diffs = std.ArrayList(Diff).init(arena.allocator()); - try diffs.appendSlice(&.{ - Diff{ .operation = .equal, .text = try arena.allocator().dupe(u8, "\u{0001}\u{0002}\u{0001}") }, - Diff{ .operation = .insert, .text = try arena.allocator().dupe(u8, "\u{0002}\u{0001}\u{0002}") }, + var diffs = DiffList{}; + defer deinitDiffList(alloc, &diffs); + try diffs.appendSlice(alloc, &.{ + Diff{ .operation = .equal, .text = try alloc.dupe(u8, "\u{0001}\u{0002}\u{0001}") }, + Diff{ .operation = .insert, .text = try alloc.dupe(u8, "\u{0002}\u{0001}\u{0002}") }, }); - var tmp_vector = std.ArrayList([]const u8).init(arena.allocator()); + var tmp_vector = std.ArrayList([]const u8).init(alloc); + defer tmp_vector.deinit(); try tmp_vector.append(""); try tmp_vector.append("alpha\n"); try tmp_vector.append("beta\n"); - try diffCharsToLines(arena.allocator(), diffs.items, tmp_vector.items); + try diffCharsToLines(alloc, diffs.items, tmp_vector.items); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ Diff.init(.equal, "alpha\nbeta\nalpha\n"), From a2beb6f4fb61bcbd51a4a9aab33c5779a7f9c46d Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 16:31:28 -0400 Subject: [PATCH 008/176] Fix for diffCleanupMerge memory I'll need to knock out all the tests in diffMergeSemantic so that I can flip the rest of the diffCleanupMerge tests on, without trying to free .rodata memory. The real annoyance of the decision to punt on memory management is in the tests, which all have to be rewritten. Whoever you are, you could at least have written the tests responsibly, and spared the person who cleans up your sloppy code from a bunch of drudgery. --- DiffMatchPatch.zig | 428 +++++++++++++++++++++++---------------------- 1 file changed, 220 insertions(+), 208 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index c34ea65..d36cfbf 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -824,14 +824,13 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo var nt = try allocator.alloc(u8, diffs.items[ii].text.len + common_length); const ot = diffs.items[ii].text; + defer allocator.free(ot); @memcpy(nt[0..ot.len], ot); @memcpy(nt[ot.len..], text_insert.items[0..common_length]); diffs.items[ii].text = nt; } else { - const text = std.ArrayListUnmanaged(u8){ - .items = try allocator.dupe(u8, text_insert.items[0..common_length]), - }; - try diffs.insert(allocator, 0, Diff.init(.equal, try allocator.dupe(u8, text.items))); + const text = try allocator.dupe(u8, text_insert.items[0..common_length]); + try diffs.insert(allocator, 0, Diff.init(.equal, text)); pointer += 1; } try text_insert.replaceRange(allocator, 0, common_length, &.{}); @@ -841,9 +840,11 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo // @ZigPort this seems very wrong common_length = diffCommonSuffix(text_insert.items, text_delete.items); if (common_length != 0) { + const old_text = diffs.items[pointer].text; + defer allocator.free(old_text); diffs.items[pointer].text = try std.mem.concat(allocator, u8, &.{ text_insert.items[text_insert.items.len - common_length ..], - diffs.items[pointer].text, + old_text, }); text_insert.items.len -= common_length; text_delete.items.len -= common_length; @@ -851,18 +852,21 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo } // Delete the offending records and add the merged ones. pointer -= count_delete + count_insert; + freeRangeDiffList(allocator, diffs, pointer, count_delete + count_insert); try diffs.replaceRange(allocator, pointer, count_delete + count_insert, &.{}); if (text_delete.items.len != 0) { - try diffs.replaceRange(allocator, pointer, 0, &.{ - Diff.init(.delete, try allocator.dupe(u8, text_delete.items)), - }); + try diffs.insert(allocator, pointer, Diff.init( + .delete, + try allocator.dupe(u8, text_delete.items), + )); pointer += 1; } if (text_insert.items.len != 0) { - try diffs.replaceRange(allocator, pointer, 0, &.{ - Diff.init(.insert, try allocator.dupe(u8, text_insert.items)), - }); + try diffs.insert(allocator, pointer, Diff.init( + .insert, + try allocator.dupe(u8, text_insert.items), + )); pointer += 1; } pointer += 1; @@ -871,10 +875,12 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo // TODO: Fix using realloc or smth var nt = try allocator.alloc(u8, diffs.items[pointer - 1].text.len + diffs.items[pointer].text.len); const ot = diffs.items[pointer - 1].text; + defer (allocator.free(ot)); @memcpy(nt[0..ot.len], ot); @memcpy(nt[ot.len..], diffs.items[pointer].text); diffs.items[pointer - 1].text = nt; - _ = diffs.orderedRemove(pointer); + const dead_diff = diffs.orderedRemove(pointer); + allocator.free(dead_diff.text); } else { pointer += 1; } @@ -912,7 +918,7 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo }); diffs.items[pointer].text = pt; diffs.items[pointer + 1].text = p1t; - // XXX reactivate freeRangeDiffList(allocator, diffs, pointer - 1, 1); + freeRangeDiffList(allocator, diffs, pointer - 1, 1); try diffs.replaceRange(allocator, pointer - 1, 1, &.{}); changes = true; } else if (std.mem.startsWith(u8, diffs.items[pointer].text, diffs.items[pointer + 1].text)) { @@ -926,7 +932,7 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo }); diffs.items[pointer - 1].text = pm1t; diffs.items[pointer].text = pt; - // XXX reactivate freeRangeDiffList(allocator, diffs, pointer - 1, 1); + freeRangeDiffList(allocator, diffs, pointer - 1, 1); try diffs.replaceRange(allocator, pointer + 1, 1, &.{}); changes = true; } @@ -1611,17 +1617,15 @@ test diffLinesToChars { } test diffCharsToLines { - var arena = std.heap.ArenaAllocator.init(testing.allocator); - defer arena.deinit(); - const alloc = std.testing.allocator; - const equal_a = Diff.init(.equal, try alloc.dupe(u8, "a")); - defer alloc.free(equal_a.text); - const insert_a = Diff.init(.insert, try alloc.dupe(u8, "a")); - defer alloc.free(insert_a.text); - const equal_b = Diff.init(.equal, try alloc.dupe(u8, "b")); - defer alloc.free(equal_b.text); - const delete_b = Diff.init(.delete, try alloc.dupe(u8, "b")); - defer alloc.free(delete_b.text); + const allocator = std.testing.allocator; + const equal_a = Diff.init(.equal, try allocator.dupe(u8, "a")); + defer allocator.free(equal_a.text); + const insert_a = Diff.init(.insert, try allocator.dupe(u8, "a")); + defer allocator.free(insert_a.text); + const equal_b = Diff.init(.equal, try allocator.dupe(u8, "b")); + defer allocator.free(equal_b.text); + const delete_b = Diff.init(.delete, try allocator.dupe(u8, "b")); + defer allocator.free(delete_b.text); try testing.expect(equal_a.eql(equal_a)); try testing.expect(!insert_a.eql(equal_a)); try testing.expect(!equal_a.eql(equal_b)); @@ -1629,17 +1633,17 @@ test diffCharsToLines { // Convert chars up to lines. var diffs = DiffList{}; - defer deinitDiffList(alloc, &diffs); - try diffs.appendSlice(alloc, &.{ - Diff{ .operation = .equal, .text = try alloc.dupe(u8, "\u{0001}\u{0002}\u{0001}") }, - Diff{ .operation = .insert, .text = try alloc.dupe(u8, "\u{0002}\u{0001}\u{0002}") }, + defer deinitDiffList(allocator, &diffs); + try diffs.appendSlice(allocator, &.{ + Diff{ .operation = .equal, .text = try allocator.dupe(u8, "\u{0001}\u{0002}\u{0001}") }, + Diff{ .operation = .insert, .text = try allocator.dupe(u8, "\u{0002}\u{0001}\u{0002}") }, }); - var tmp_vector = std.ArrayList([]const u8).init(alloc); + var tmp_vector = std.ArrayList([]const u8).init(allocator); defer tmp_vector.deinit(); try tmp_vector.append(""); try tmp_vector.append("alpha\n"); try tmp_vector.append("beta\n"); - try diffCharsToLines(alloc, diffs.items, tmp_vector.items); + try diffCharsToLines(allocator, diffs.items, tmp_vector.items); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ Diff.init(.equal, "alpha\nbeta\nalpha\n"), @@ -1653,188 +1657,196 @@ test diffCleanupMerge { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); + const alloc = std.testing.allocator; // Cleanup a messy diff. var diffs = DiffList{}; - try testing.expectEqualDeep(@as([]const Diff, &[0]Diff{}), diffs.items); // Null case - - try diffs.appendSlice(arena.allocator(), &[_]Diff{ - .{ .operation = .equal, .text = "a" }, - .{ .operation = .delete, .text = "b" }, - .{ .operation = .insert, .text = "c" }, - }); - try diffCleanupMerge(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .equal, .text = "a" }, - .{ .operation = .delete, .text = "b" }, - .{ .operation = .insert, .text = "c" }, - }), diffs.items); // No change case - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &[_]Diff{ - .{ .operation = .equal, .text = "a" }, - .{ .operation = .equal, .text = "b" }, - .{ .operation = .equal, .text = "c" }, - }); - try diffCleanupMerge(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .equal, .text = "abc" }, - }), diffs.items); // Merge equalities - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &[_]Diff{ - .{ .operation = .delete, .text = "a" }, - .{ .operation = .delete, .text = "b" }, - .{ .operation = .delete, .text = "c" }, - }); - try diffCleanupMerge(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .delete, .text = "abc" }, - }), diffs.items); // Merge deletions - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &[_]Diff{ - .{ .operation = .insert, .text = "a" }, - .{ .operation = .insert, .text = "b" }, - .{ .operation = .insert, .text = "c" }, - }); - try diffCleanupMerge(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .insert, .text = "abc" }, - }), diffs.items); // Merge insertions - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &[_]Diff{ - .{ .operation = .delete, .text = "a" }, - .{ .operation = .insert, .text = "b" }, - .{ .operation = .delete, .text = "c" }, - .{ .operation = .insert, .text = "d" }, - .{ .operation = .equal, .text = "e" }, - .{ .operation = .equal, .text = "f" }, - }); - try diffCleanupMerge(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .delete, .text = "ac" }, - .{ .operation = .insert, .text = "bd" }, - .{ .operation = .equal, .text = "ef" }, - }), diffs.items); // Merge interweave - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &[_]Diff{ - .{ .operation = .delete, .text = "a" }, - .{ .operation = .insert, .text = "abc" }, - .{ .operation = .delete, .text = "dc" }, - }); - try diffCleanupMerge(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .equal, .text = "a" }, - .{ .operation = .delete, .text = "d" }, - .{ .operation = .insert, .text = "b" }, - .{ .operation = .equal, .text = "c" }, - }), diffs.items); // Prefix and suffix detection - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &[_]Diff{ - .{ .operation = .equal, .text = "x" }, - .{ .operation = .delete, .text = "a" }, - .{ .operation = .insert, .text = "abc" }, - .{ .operation = .delete, .text = "dc" }, - .{ .operation = .equal, .text = "y" }, - }); - try diffCleanupMerge(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .equal, .text = "xa" }, - .{ .operation = .delete, .text = "d" }, - .{ .operation = .insert, .text = "b" }, - .{ .operation = .equal, .text = "cy" }, - }), diffs.items); // Prefix and suffix detection with equalities - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &[_]Diff{ - .{ .operation = .equal, .text = "a" }, - .{ .operation = .insert, .text = "ba" }, - .{ .operation = .equal, .text = "c" }, - }); - try diffCleanupMerge(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .insert, .text = "ab" }, - .{ .operation = .equal, .text = "ac" }, - }), diffs.items); // Slide edit left - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &[_]Diff{ - .{ .operation = .equal, .text = "c" }, - .{ .operation = .insert, .text = "ab" }, - .{ .operation = .equal, .text = "a" }, - }); - try diffCleanupMerge(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .equal, .text = "ca" }, - .{ .operation = .insert, .text = "ba" }, - }), diffs.items); // Slide edit right - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &[_]Diff{ - Diff.init(.equal, "a"), - Diff.init(.delete, "b"), - Diff.init(.equal, "c"), - Diff.init(.delete, "ac"), - Diff.init(.equal, "x"), - }); - try diffCleanupMerge(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - Diff.init(.delete, "abc"), - Diff.init(.equal, "acx"), - }), diffs.items); // Slide edit left recursive - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &[_]Diff{ - Diff.init(.equal, "x"), - Diff.init(.delete, "ca"), - Diff.init(.equal, "c"), - Diff.init(.delete, "b"), - Diff.init(.equal, "a"), - }); - try diffCleanupMerge(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - Diff.init(.equal, "xca"), - Diff.init(.delete, "cba"), - }), diffs.items); // Slide edit right recursive - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &[_]Diff{ - Diff.init(.delete, "b"), - Diff.init(.insert, "ab"), - Diff.init(.equal, "c"), - }); - try diffCleanupMerge(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - Diff.init(.insert, "a"), - Diff.init(.equal, "bc"), - }), diffs.items); // Empty merge + defer deinitDiffList(alloc, &diffs); - diffs.items.len = 0; + try testing.expectEqualDeep(@as([]const Diff, &[0]Diff{}), diffs.items); // Null case - try diffs.appendSlice(arena.allocator(), &[_]Diff{ - Diff.init(.equal, ""), - Diff.init(.insert, "a"), - Diff.init(.equal, "b"), + try diffs.appendSlice(alloc, &[_]Diff{ + .{ + .operation = .equal, + .text = try alloc.dupe(u8, "a"), + }, + .{ + .operation = .delete, + .text = try alloc.dupe(u8, "b"), + }, + .{ + .operation = .insert, + .text = try alloc.dupe(u8, "c"), + }, }); - try diffCleanupMerge(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - Diff.init(.insert, "a"), - Diff.init(.equal, "b"), - }), diffs.items); // Empty equality + try diffCleanupMerge(alloc, &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "b" }, .{ .operation = .insert, .text = "c" } }), diffs.items); // No change case + // + // var diffs2 = DiffList{}; + // + // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // .{ .operation = .equal, .text = "a" }, + // .{ .operation = .equal, .text = "b" }, + // .{ .operation = .equal, .text = "c" }, + // }); + // try diffCleanupMerge(arena.allocator(), &diffs); + // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + // .{ .operation = .equal, .text = "abc" }, + // }), diffs2.items); // Merge equalities + // + // diffs2.items.len = 0; + // + // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // .{ .operation = .delete, .text = "a" }, + // .{ .operation = .delete, .text = "b" }, + // .{ .operation = .delete, .text = "c" }, + // }); + // try diffCleanupMerge(arena.allocator(), &diffs); + // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + // .{ .operation = .delete, .text = "abc" }, + // }), diffs2.items); // Merge deletions + // + // diffs2.items.len = 0; + // + // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // .{ .operation = .insert, .text = "a" }, + // .{ .operation = .insert, .text = "b" }, + // .{ .operation = .insert, .text = "c" }, + // }); + // try diffCleanupMerge(arena.allocator(), &diffs); + // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + // .{ .operation = .insert, .text = "abc" }, + // }), diffs2.items); // Merge insertions + // + // diffs2.items.len = 0; + // + // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // .{ .operation = .delete, .text = "a" }, + // .{ .operation = .insert, .text = "b" }, + // .{ .operation = .delete, .text = "c" }, + // .{ .operation = .insert, .text = "d" }, + // .{ .operation = .equal, .text = "e" }, + // .{ .operation = .equal, .text = "f" }, + // }); + // try diffCleanupMerge(arena.allocator(), &diffs); + // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + // .{ .operation = .delete, .text = "ac" }, + // .{ .operation = .insert, .text = "bd" }, + // .{ .operation = .equal, .text = "ef" }, + // }), diffs2.items); // Merge interweave + // + // diffs2.items.len = 0; + // + // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // .{ .operation = .delete, .text = "a" }, + // .{ .operation = .insert, .text = "abc" }, + // .{ .operation = .delete, .text = "dc" }, + // }); + // try diffCleanupMerge(arena.allocator(), &diffs); + // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + // .{ .operation = .equal, .text = "a" }, + // .{ .operation = .delete, .text = "d" }, + // .{ .operation = .insert, .text = "b" }, + // .{ .operation = .equal, .text = "c" }, + // }), diffs2.items); // Prefix and suffix detection + // + // diffs2.items.len = 0; + // + // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // .{ .operation = .equal, .text = "x" }, + // .{ .operation = .delete, .text = "a" }, + // .{ .operation = .insert, .text = "abc" }, + // .{ .operation = .delete, .text = "dc" }, + // .{ .operation = .equal, .text = "y" }, + // }); + // try diffCleanupMerge(arena.allocator(), &diffs); + // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + // .{ .operation = .equal, .text = "xa" }, + // .{ .operation = .delete, .text = "d" }, + // .{ .operation = .insert, .text = "b" }, + // .{ .operation = .equal, .text = "cy" }, + // }), diffs2.items); // Prefix and suffix detection with equalities + // + // diffs2.items.len = 0; + // + // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // .{ .operation = .equal, .text = "a" }, + // .{ .operation = .insert, .text = "ba" }, + // .{ .operation = .equal, .text = "c" }, + // }); + // try diffCleanupMerge(arena.allocator(), &diffs); + // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + // .{ .operation = .insert, .text = "ab" }, + // .{ .operation = .equal, .text = "ac" }, + // }), diffs2.items); // Slide edit left + // + // diffs2.items.len = 0; + // + // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // .{ .operation = .equal, .text = "c" }, + // .{ .operation = .insert, .text = "ab" }, + // .{ .operation = .equal, .text = "a" }, + // }); + // try diffCleanupMerge(arena.allocator(), &diffs); + // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + // .{ .operation = .equal, .text = "ca" }, + // .{ .operation = .insert, .text = "ba" }, + // }), diffs2.items); // Slide edit right + // + // diffs2.items.len = 0; + // + // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // Diff.init(.equal, "a"), + // Diff.init(.delete, "b"), + // Diff.init(.equal, "c"), + // Diff.init(.delete, "ac"), + // Diff.init(.equal, "x"), + // }); + // try diffCleanupMerge(arena.allocator(), &diffs); + // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + // Diff.init(.delete, "abc"), + // Diff.init(.equal, "acx"), + // }), diffs2.items); // Slide edit left recursive + // + // diffs2.items.len = 0; + // + // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // Diff.init(.equal, "x"), + // Diff.init(.delete, "ca"), + // Diff.init(.equal, "c"), + // Diff.init(.delete, "b"), + // Diff.init(.equal, "a"), + // }); + // try diffCleanupMerge(arena.allocator(), &diffs); + // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + // Diff.init(.equal, "xca"), + // Diff.init(.delete, "cba"), + // }), diffs2.items); // Slide edit right recursive + // + // diffs2.items.len = 0; + // + // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // Diff.init(.delete, "b"), + // Diff.init(.insert, "ab"), + // Diff.init(.equal, "c"), + // }); + // try diffCleanupMerge(arena.allocator(), &diffs); + // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + // Diff.init(.insert, "a"), + // Diff.init(.equal, "bc"), + // }), diffs2.items); // Empty merge + // + // diffs2.items.len = 0; + // + // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // Diff.init(.equal, ""), + // Diff.init(.insert, "a"), + // Diff.init(.equal, "b"), + // }); + // try diffCleanupMerge(arena.allocator(), &diffs); + // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + // Diff.init(.insert, "a"), + // Diff.init(.equal, "b"), + // }), diffs2.items); // Empty equality } test diffCleanupSemanticLossless { From 7c5cccc615d5c1279b8285ef2e34a4d80513a445 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 17:12:33 -0400 Subject: [PATCH 009/176] One test at a time... --- DiffMatchPatch.zig | 558 +++++++++++++++++++++++---------------------- 1 file changed, 286 insertions(+), 272 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index d36cfbf..0f0ead8 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -852,21 +852,23 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo } // Delete the offending records and add the merged ones. pointer -= count_delete + count_insert; - freeRangeDiffList(allocator, diffs, pointer, count_delete + count_insert); - try diffs.replaceRange(allocator, pointer, count_delete + count_insert, &.{}); + if (count_delete + count_insert > 0) { + freeRangeDiffList(allocator, diffs, pointer, count_delete + count_insert); + try diffs.replaceRange(allocator, pointer, count_delete + count_insert, &.{}); + } if (text_delete.items.len != 0) { - try diffs.insert(allocator, pointer, Diff.init( - .delete, - try allocator.dupe(u8, text_delete.items), - )); + allocator.free(diffs.items[pointer].text); + try diffs.replaceRange(allocator, pointer, 0, &.{ + Diff.init(.delete, try allocator.dupe(u8, text_delete.items)), + }); pointer += 1; } if (text_insert.items.len != 0) { - try diffs.insert(allocator, pointer, Diff.init( - .insert, - try allocator.dupe(u8, text_insert.items), - )); + allocator.free(diffs.items[pointer].text); + try diffs.replaceRange(allocator, pointer, 0, &.{ + Diff.init(.insert, try allocator.dupe(u8, text_insert.items)), + }); pointer += 1; } pointer += 1; @@ -1680,18 +1682,26 @@ test diffCleanupMerge { }); try diffCleanupMerge(alloc, &diffs); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "b" }, .{ .operation = .insert, .text = "c" } }), diffs.items); // No change case - // - // var diffs2 = DiffList{}; - // - // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ - // .{ .operation = .equal, .text = "a" }, - // .{ .operation = .equal, .text = "b" }, - // .{ .operation = .equal, .text = "c" }, - // }); - // try diffCleanupMerge(arena.allocator(), &diffs); - // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - // .{ .operation = .equal, .text = "abc" }, - // }), diffs2.items); // Merge equalities + var diffs2 = DiffList{}; + defer deinitDiffList(alloc, &diffs2); + try diffs2.appendSlice(alloc, &[_]Diff{ + .{ + .operation = .equal, + .text = try alloc.dupe(u8, "a"), + }, + .{ + .operation = .equal, + .text = try alloc.dupe(u8, "b"), + }, + .{ + .operation = .equal, + .text = try alloc.dupe(u8, "c"), + }, + }); + try diffCleanupMerge(alloc, &diffs2); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + .{ .operation = .equal, .text = "abc" }, + }), diffs2.items); // Merge equalities // // diffs2.items.len = 0; // @@ -1850,108 +1860,110 @@ test diffCleanupMerge { } test diffCleanupSemanticLossless { - var arena = std.heap.ArenaAllocator.init(testing.allocator); - defer arena.deinit(); - - var diffs = DiffList{}; - try diffCleanupSemanticLossless(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[0]Diff{}), diffs.items); // Null case - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.equal, "AAA\r\n\r\nBBB"), - Diff.init(.insert, "\r\nDDD\r\n\r\nBBB"), - Diff.init(.equal, "\r\nEEE"), - }); - try diffCleanupSemanticLossless(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "AAA\r\n\r\n"), - Diff.init(.insert, "BBB\r\nDDD\r\n\r\n"), - Diff.init(.equal, "BBB\r\nEEE"), - }), diffs.items); - - diffs.items.len = 0; + if (false) { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.equal, "AAA\r\nBBB"), - Diff.init(.insert, " DDD\r\nBBB"), - Diff.init(.equal, " EEE"), - }); - try diffCleanupSemanticLossless(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "AAA\r\n"), - Diff.init(.insert, "BBB DDD\r\n"), - Diff.init(.equal, "BBB EEE"), - }), diffs.items); - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.equal, "The c"), - Diff.init(.insert, "ow and the c"), - Diff.init(.equal, "at."), - }); - try diffCleanupSemanticLossless(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "The "), - Diff.init(.insert, "cow and the "), - Diff.init(.equal, "cat."), - }), diffs.items); - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.equal, "The-c"), - Diff.init(.insert, "ow-and-the-c"), - Diff.init(.equal, "at."), - }); - try diffCleanupSemanticLossless(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "The-"), - Diff.init(.insert, "cow-and-the-"), - Diff.init(.equal, "cat."), - }), diffs.items); - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.equal, "a"), - Diff.init(.delete, "a"), - Diff.init(.equal, "ax"), - }); - try diffCleanupSemanticLossless(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.delete, "a"), - Diff.init(.equal, "aax"), - }), diffs.items); - - diffs.items.len = 0; - - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.equal, "xa"), - Diff.init(.delete, "a"), - Diff.init(.equal, "a"), - }); - try diffCleanupSemanticLossless(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "xaa"), - Diff.init(.delete, "a"), - }), diffs.items); + var diffs = DiffList{}; + try diffCleanupSemanticLossless(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[0]Diff{}), diffs.items); // Null case - diffs.items.len = 0; + diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.equal, "The xxx. The "), - Diff.init(.insert, "zzz. The "), - Diff.init(.equal, "yyy."), - }); - try diffCleanupSemanticLossless(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "The xxx."), - Diff.init(.insert, " The zzz."), - Diff.init(.equal, " The yyy."), - }), diffs.items); + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.equal, "AAA\r\n\r\nBBB"), + Diff.init(.insert, "\r\nDDD\r\n\r\nBBB"), + Diff.init(.equal, "\r\nEEE"), + }); + try diffCleanupSemanticLossless(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "AAA\r\n\r\n"), + Diff.init(.insert, "BBB\r\nDDD\r\n\r\n"), + Diff.init(.equal, "BBB\r\nEEE"), + }), diffs.items); + + diffs.items.len = 0; + + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.equal, "AAA\r\nBBB"), + Diff.init(.insert, " DDD\r\nBBB"), + Diff.init(.equal, " EEE"), + }); + try diffCleanupSemanticLossless(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "AAA\r\n"), + Diff.init(.insert, "BBB DDD\r\n"), + Diff.init(.equal, "BBB EEE"), + }), diffs.items); + + diffs.items.len = 0; + + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.equal, "The c"), + Diff.init(.insert, "ow and the c"), + Diff.init(.equal, "at."), + }); + try diffCleanupSemanticLossless(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "The "), + Diff.init(.insert, "cow and the "), + Diff.init(.equal, "cat."), + }), diffs.items); + + diffs.items.len = 0; + + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.equal, "The-c"), + Diff.init(.insert, "ow-and-the-c"), + Diff.init(.equal, "at."), + }); + try diffCleanupSemanticLossless(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "The-"), + Diff.init(.insert, "cow-and-the-"), + Diff.init(.equal, "cat."), + }), diffs.items); + + diffs.items.len = 0; + + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.equal, "a"), + Diff.init(.delete, "a"), + Diff.init(.equal, "ax"), + }); + try diffCleanupSemanticLossless(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.delete, "a"), + Diff.init(.equal, "aax"), + }), diffs.items); + + diffs.items.len = 0; + + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.equal, "xa"), + Diff.init(.delete, "a"), + Diff.init(.equal, "a"), + }); + try diffCleanupSemanticLossless(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "xaa"), + Diff.init(.delete, "a"), + }), diffs.items); + + diffs.items.len = 0; + + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.equal, "The xxx. The "), + Diff.init(.insert, "zzz. The "), + Diff.init(.equal, "yyy."), + }); + try diffCleanupSemanticLossless(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "The xxx."), + Diff.init(.insert, " The zzz."), + Diff.init(.equal, " The yyy."), + }), diffs.items); + } } fn rebuildtexts(allocator: std.mem.Allocator, diffs: DiffList) ![2][]const u8 { @@ -2116,155 +2128,157 @@ test diff { } test diffCleanupSemantic { - var arena = std.heap.ArenaAllocator.init(talloc); - defer arena.deinit(); - - // Cleanup semantically trivial equalities. - // Null case. - var diffs = DiffList{}; - defer diffs.deinit(arena.allocator()); - // var this = default; - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqual(@as(usize, 0), diffs.items.len); // Null case - - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "ab"), - Diff.init(.insert, "cd"), - Diff.init(.equal, "12"), - Diff.init(.delete, "e"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // No elimination #1 - Diff.init(.delete, "ab"), - Diff.init(.insert, "cd"), - Diff.init(.equal, "12"), - Diff.init(.delete, "e"), - }), diffs.items); - - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "abc"), - Diff.init(.insert, "ABC"), - Diff.init(.equal, "1234"), - Diff.init(.delete, "wxyz"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // No elimination #2 - Diff.init(.delete, "abc"), - Diff.init(.insert, "ABC"), - Diff.init(.equal, "1234"), - Diff.init(.delete, "wxyz"), - }), diffs.items); - - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "a"), - Diff.init(.equal, "b"), - Diff.init(.delete, "c"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Simple elimination - Diff.init(.delete, "abc"), - Diff.init(.insert, "b"), - }), diffs.items); - - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "ab"), - Diff.init(.equal, "cd"), - Diff.init(.delete, "e"), - Diff.init(.equal, "f"), - Diff.init(.insert, "g"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Backpass elimination - Diff.init(.delete, "abcdef"), - Diff.init(.insert, "cdfg"), - }), diffs.items); - - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.insert, "1"), - Diff.init(.equal, "A"), - Diff.init(.delete, "B"), - Diff.init(.insert, "2"), - Diff.init(.equal, "_"), - Diff.init(.insert, "1"), - Diff.init(.equal, "A"), - Diff.init(.delete, "B"), - Diff.init(.insert, "2"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Multiple elimination - Diff.init(.delete, "AB_AB"), - Diff.init(.insert, "1A2_1A2"), - }), diffs.items); - - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.equal, "The c"), - Diff.init(.delete, "ow and the c"), - Diff.init(.equal, "at."), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Word boundaries - Diff.init(.equal, "The "), - Diff.init(.delete, "cow and the "), - Diff.init(.equal, "cat."), - }), diffs.items); - - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "abcxx"), - Diff.init(.insert, "xxdef"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // No overlap elimination - Diff.init(.delete, "abcxx"), - Diff.init(.insert, "xxdef"), - }), diffs.items); - - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "abcxxx"), - Diff.init(.insert, "xxxdef"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Overlap elimination - Diff.init(.delete, "abc"), - Diff.init(.equal, "xxx"), - Diff.init(.insert, "def"), - }), diffs.items); - - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "xxxabc"), - Diff.init(.insert, "defxxx"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Reverse overlap elimination - Diff.init(.insert, "def"), - Diff.init(.equal, "xxx"), - Diff.init(.delete, "abc"), - }), diffs.items); - - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "abcd1212"), - Diff.init(.insert, "1212efghi"), - Diff.init(.equal, "----"), - Diff.init(.delete, "A3"), - Diff.init(.insert, "3BC"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Two overlap eliminations - Diff.init(.delete, "abcd"), - Diff.init(.equal, "1212"), - Diff.init(.insert, "efghi"), - Diff.init(.equal, "----"), - Diff.init(.delete, "A"), - Diff.init(.equal, "3"), - Diff.init(.insert, "BC"), - }), diffs.items); + if (false) { + var arena = std.heap.ArenaAllocator.init(talloc); + defer arena.deinit(); + + // Cleanup semantically trivial equalities. + // Null case. + var diffs = DiffList{}; + defer diffs.deinit(arena.allocator()); + // var this = default; + try diffCleanupSemantic(arena.allocator(), &diffs); + try testing.expectEqual(@as(usize, 0), diffs.items.len); // Null case + + diffs.items.len = 0; + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.delete, "ab"), + Diff.init(.insert, "cd"), + Diff.init(.equal, "12"), + Diff.init(.delete, "e"), + }); + try diffCleanupSemantic(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // No elimination #1 + Diff.init(.delete, "ab"), + Diff.init(.insert, "cd"), + Diff.init(.equal, "12"), + Diff.init(.delete, "e"), + }), diffs.items); + + diffs.items.len = 0; + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.delete, "abc"), + Diff.init(.insert, "ABC"), + Diff.init(.equal, "1234"), + Diff.init(.delete, "wxyz"), + }); + try diffCleanupSemantic(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // No elimination #2 + Diff.init(.delete, "abc"), + Diff.init(.insert, "ABC"), + Diff.init(.equal, "1234"), + Diff.init(.delete, "wxyz"), + }), diffs.items); + + diffs.items.len = 0; + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.delete, "a"), + Diff.init(.equal, "b"), + Diff.init(.delete, "c"), + }); + try diffCleanupSemantic(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Simple elimination + Diff.init(.delete, "abc"), + Diff.init(.insert, "b"), + }), diffs.items); + + diffs.items.len = 0; + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.delete, "ab"), + Diff.init(.equal, "cd"), + Diff.init(.delete, "e"), + Diff.init(.equal, "f"), + Diff.init(.insert, "g"), + }); + try diffCleanupSemantic(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Backpass elimination + Diff.init(.delete, "abcdef"), + Diff.init(.insert, "cdfg"), + }), diffs.items); + + diffs.items.len = 0; + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.insert, "1"), + Diff.init(.equal, "A"), + Diff.init(.delete, "B"), + Diff.init(.insert, "2"), + Diff.init(.equal, "_"), + Diff.init(.insert, "1"), + Diff.init(.equal, "A"), + Diff.init(.delete, "B"), + Diff.init(.insert, "2"), + }); + try diffCleanupSemantic(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Multiple elimination + Diff.init(.delete, "AB_AB"), + Diff.init(.insert, "1A2_1A2"), + }), diffs.items); + + diffs.items.len = 0; + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.equal, "The c"), + Diff.init(.delete, "ow and the c"), + Diff.init(.equal, "at."), + }); + try diffCleanupSemantic(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Word boundaries + Diff.init(.equal, "The "), + Diff.init(.delete, "cow and the "), + Diff.init(.equal, "cat."), + }), diffs.items); + + diffs.items.len = 0; + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.delete, "abcxx"), + Diff.init(.insert, "xxdef"), + }); + try diffCleanupSemantic(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // No overlap elimination + Diff.init(.delete, "abcxx"), + Diff.init(.insert, "xxdef"), + }), diffs.items); + + diffs.items.len = 0; + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.delete, "abcxxx"), + Diff.init(.insert, "xxxdef"), + }); + try diffCleanupSemantic(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Overlap elimination + Diff.init(.delete, "abc"), + Diff.init(.equal, "xxx"), + Diff.init(.insert, "def"), + }), diffs.items); + + diffs.items.len = 0; + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.delete, "xxxabc"), + Diff.init(.insert, "defxxx"), + }); + try diffCleanupSemantic(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Reverse overlap elimination + Diff.init(.insert, "def"), + Diff.init(.equal, "xxx"), + Diff.init(.delete, "abc"), + }), diffs.items); + + diffs.items.len = 0; + try diffs.appendSlice(arena.allocator(), &.{ + Diff.init(.delete, "abcd1212"), + Diff.init(.insert, "1212efghi"), + Diff.init(.equal, "----"), + Diff.init(.delete, "A3"), + Diff.init(.insert, "3BC"), + }); + try diffCleanupSemantic(arena.allocator(), &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Two overlap eliminations + Diff.init(.delete, "abcd"), + Diff.init(.equal, "1212"), + Diff.init(.insert, "efghi"), + Diff.init(.equal, "----"), + Diff.init(.delete, "A"), + Diff.init(.equal, "3"), + Diff.init(.insert, "BC"), + }), diffs.items); + } } From dbc3a84e6935b6d43be0396a06194e18e87fa5fe Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 17:28:19 -0400 Subject: [PATCH 010/176] More tests Found a double free in the falsed-out one, so that's next up... --- DiffMatchPatch.zig | 155 ++++++++++++++++++++++++++++----------------- 1 file changed, 96 insertions(+), 59 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 0f0ead8..e9ab5d1 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1702,56 +1702,93 @@ test diffCleanupMerge { try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .equal, .text = "abc" }, }), diffs2.items); // Merge equalities + + var diffs3 = DiffList{}; + defer deinitDiffList(alloc, &diffs3); + + try diffs3.appendSlice(alloc, &[_]Diff{ + .{ + .operation = .delete, + .text = try alloc.dupe(u8, "a"), + }, + .{ + .operation = .delete, + .text = try alloc.dupe(u8, "b"), + }, + .{ + .operation = .delete, + .text = try alloc.dupe(u8, "c"), + }, + }); + try diffCleanupMerge(alloc, &diffs3); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + .{ .operation = .delete, .text = "abc" }, + }), diffs3.items); // Merge deletions + + var diffs4 = DiffList{}; + defer deinitDiffList(alloc, &diffs4); + try diffs4.appendSlice(alloc, &[_]Diff{ + .{ + .operation = .insert, + .text = try alloc.dupe(u8, "a"), + }, + .{ + .operation = .insert, + .text = try alloc.dupe(u8, "b"), + }, + .{ + .operation = .insert, + .text = try alloc.dupe(u8, "c"), + }, + }); + try diffCleanupMerge(alloc, &diffs4); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + .{ .operation = .insert, .text = "abc" }, + }), diffs4.items); // Merge insertions + + if (false) { + var diffs5 = DiffList{}; + defer deinitDiffList(alloc, &diffs5); + try diffs5.appendSlice(alloc, &[_]Diff{ + .{ + .operation = .delete, + .text = try alloc.dupe(u8, "a"), + }, + .{ + .operation = .insert, + .text = try alloc.dupe(u8, "b"), + }, + .{ + .operation = .delete, + .text = try alloc.dupe(u8, "c"), + }, + .{ + .operation = .insert, + .text = try alloc.dupe(u8, "d"), + }, + .{ + .operation = .equal, + .text = try alloc.dupe(u8, "e"), + }, + .{ + .operation = .equal, + .text = try alloc.dupe(u8, "f"), + }, + }); + try diffCleanupMerge(alloc, &diffs5); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + .{ .operation = .delete, .text = "ac" }, + .{ .operation = .insert, .text = "bd" }, + .{ .operation = .equal, .text = "ef" }, + }), diffs5.items); // Merge interweave + } // - // diffs2.items.len = 0; - // - // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ - // .{ .operation = .delete, .text = "a" }, - // .{ .operation = .delete, .text = "b" }, - // .{ .operation = .delete, .text = "c" }, - // }); - // try diffCleanupMerge(arena.allocator(), &diffs); - // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - // .{ .operation = .delete, .text = "abc" }, - // }), diffs2.items); // Merge deletions - // - // diffs2.items.len = 0; - // - // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ - // .{ .operation = .insert, .text = "a" }, - // .{ .operation = .insert, .text = "b" }, - // .{ .operation = .insert, .text = "c" }, - // }); - // try diffCleanupMerge(arena.allocator(), &diffs); - // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - // .{ .operation = .insert, .text = "abc" }, - // }), diffs2.items); // Merge insertions - // - // diffs2.items.len = 0; - // - // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ - // .{ .operation = .delete, .text = "a" }, - // .{ .operation = .insert, .text = "b" }, - // .{ .operation = .delete, .text = "c" }, - // .{ .operation = .insert, .text = "d" }, - // .{ .operation = .equal, .text = "e" }, - // .{ .operation = .equal, .text = "f" }, - // }); - // try diffCleanupMerge(arena.allocator(), &diffs); - // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - // .{ .operation = .delete, .text = "ac" }, - // .{ .operation = .insert, .text = "bd" }, - // .{ .operation = .equal, .text = "ef" }, - // }), diffs2.items); // Merge interweave - // - // diffs2.items.len = 0; - // - // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // try diffs2.appendSlice(alloc, &[_]Diff{ // .{ .operation = .delete, .text = "a" }, // .{ .operation = .insert, .text = "abc" }, // .{ .operation = .delete, .text = "dc" }, // }); - // try diffCleanupMerge(arena.allocator(), &diffs); + // try diffCleanupMerge(alloc, &diffs2); // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // .{ .operation = .equal, .text = "a" }, // .{ .operation = .delete, .text = "d" }, @@ -1761,14 +1798,14 @@ test diffCleanupMerge { // // diffs2.items.len = 0; // - // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // try diffs2.appendSlice(alloc, &[_]Diff{ // .{ .operation = .equal, .text = "x" }, // .{ .operation = .delete, .text = "a" }, // .{ .operation = .insert, .text = "abc" }, // .{ .operation = .delete, .text = "dc" }, // .{ .operation = .equal, .text = "y" }, // }); - // try diffCleanupMerge(arena.allocator(), &diffs); + // try diffCleanupMerge(alloc, &diffs2); // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // .{ .operation = .equal, .text = "xa" }, // .{ .operation = .delete, .text = "d" }, @@ -1778,12 +1815,12 @@ test diffCleanupMerge { // // diffs2.items.len = 0; // - // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // try diffs2.appendSlice(alloc, &[_]Diff{ // .{ .operation = .equal, .text = "a" }, // .{ .operation = .insert, .text = "ba" }, // .{ .operation = .equal, .text = "c" }, // }); - // try diffCleanupMerge(arena.allocator(), &diffs); + // try diffCleanupMerge(alloc, &diffs2); // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // .{ .operation = .insert, .text = "ab" }, // .{ .operation = .equal, .text = "ac" }, @@ -1791,12 +1828,12 @@ test diffCleanupMerge { // // diffs2.items.len = 0; // - // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // try diffs2.appendSlice(alloc, &[_]Diff{ // .{ .operation = .equal, .text = "c" }, // .{ .operation = .insert, .text = "ab" }, // .{ .operation = .equal, .text = "a" }, // }); - // try diffCleanupMerge(arena.allocator(), &diffs); + // try diffCleanupMerge(alloc, &diffs2); // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // .{ .operation = .equal, .text = "ca" }, // .{ .operation = .insert, .text = "ba" }, @@ -1804,14 +1841,14 @@ test diffCleanupMerge { // // diffs2.items.len = 0; // - // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // try diffs2.appendSlice(alloc, &[_]Diff{ // Diff.init(.equal, "a"), // Diff.init(.delete, "b"), // Diff.init(.equal, "c"), // Diff.init(.delete, "ac"), // Diff.init(.equal, "x"), // }); - // try diffCleanupMerge(arena.allocator(), &diffs); + // try diffCleanupMerge(alloc, &diffs2); // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Diff.init(.delete, "abc"), // Diff.init(.equal, "acx"), @@ -1819,14 +1856,14 @@ test diffCleanupMerge { // // diffs2.items.len = 0; // - // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // try diffs2.appendSlice(alloc, &[_]Diff{ // Diff.init(.equal, "x"), // Diff.init(.delete, "ca"), // Diff.init(.equal, "c"), // Diff.init(.delete, "b"), // Diff.init(.equal, "a"), // }); - // try diffCleanupMerge(arena.allocator(), &diffs); + // try diffCleanupMerge(alloc, &diffs2); // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Diff.init(.equal, "xca"), // Diff.init(.delete, "cba"), @@ -1834,12 +1871,12 @@ test diffCleanupMerge { // // diffs2.items.len = 0; // - // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // try diffs2.appendSlice(alloc, &[_]Diff{ // Diff.init(.delete, "b"), // Diff.init(.insert, "ab"), // Diff.init(.equal, "c"), // }); - // try diffCleanupMerge(arena.allocator(), &diffs); + // try diffCleanupMerge(alloc, &diffs2); // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Diff.init(.insert, "a"), // Diff.init(.equal, "bc"), @@ -1847,12 +1884,12 @@ test diffCleanupMerge { // // diffs2.items.len = 0; // - // try diffs2.appendSlice(arena.allocator(), &[_]Diff{ + // try diffs2.appendSlice(alloc, &[_]Diff{ // Diff.init(.equal, ""), // Diff.init(.insert, "a"), // Diff.init(.equal, "b"), // }); - // try diffCleanupMerge(arena.allocator(), &diffs); + // try diffCleanupMerge(alloc, &diffs2); // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Diff.init(.insert, "a"), // Diff.init(.equal, "b"), From 0b2f274e99418fbcd4587f0d8c45074d5db4cfc2 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 18:05:58 -0400 Subject: [PATCH 011/176] Fix two double-frees Also changing the use of .replaceRange, which lead me to think that the item at the pointer location was being, y'know, replaced, with a use of .insert, which is,, correct. --- DiffMatchPatch.zig | 88 ++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index e9ab5d1..5d764e7 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -858,17 +858,17 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo } if (text_delete.items.len != 0) { - allocator.free(diffs.items[pointer].text); - try diffs.replaceRange(allocator, pointer, 0, &.{ - Diff.init(.delete, try allocator.dupe(u8, text_delete.items)), - }); + try diffs.insert(allocator, pointer, Diff.init( + .delete, + try allocator.dupe(u8, text_delete.items), + )); pointer += 1; } if (text_insert.items.len != 0) { - allocator.free(diffs.items[pointer].text); - try diffs.replaceRange(allocator, pointer, 0, &.{ - Diff.init(.insert, try allocator.dupe(u8, text_insert.items)), - }); + try diffs.insert(allocator, pointer, Diff.init( + .insert, + try allocator.dupe(u8, text_insert.items), + )); pointer += 1; } pointer += 1; @@ -934,7 +934,7 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo }); diffs.items[pointer - 1].text = pm1t; diffs.items[pointer].text = pt; - freeRangeDiffList(allocator, diffs, pointer - 1, 1); + freeRangeDiffList(allocator, diffs, pointer + 1, 1); try diffs.replaceRange(allocator, pointer + 1, 1, &.{}); changes = true; } @@ -1746,42 +1746,40 @@ test diffCleanupMerge { .{ .operation = .insert, .text = "abc" }, }), diffs4.items); // Merge insertions - if (false) { - var diffs5 = DiffList{}; - defer deinitDiffList(alloc, &diffs5); - try diffs5.appendSlice(alloc, &[_]Diff{ - .{ - .operation = .delete, - .text = try alloc.dupe(u8, "a"), - }, - .{ - .operation = .insert, - .text = try alloc.dupe(u8, "b"), - }, - .{ - .operation = .delete, - .text = try alloc.dupe(u8, "c"), - }, - .{ - .operation = .insert, - .text = try alloc.dupe(u8, "d"), - }, - .{ - .operation = .equal, - .text = try alloc.dupe(u8, "e"), - }, - .{ - .operation = .equal, - .text = try alloc.dupe(u8, "f"), - }, - }); - try diffCleanupMerge(alloc, &diffs5); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .delete, .text = "ac" }, - .{ .operation = .insert, .text = "bd" }, - .{ .operation = .equal, .text = "ef" }, - }), diffs5.items); // Merge interweave - } + var diffs5 = DiffList{}; + defer deinitDiffList(alloc, &diffs5); + try diffs5.appendSlice(alloc, &[_]Diff{ + .{ + .operation = .delete, + .text = try alloc.dupe(u8, "a"), + }, + .{ + .operation = .insert, + .text = try alloc.dupe(u8, "b"), + }, + .{ + .operation = .delete, + .text = try alloc.dupe(u8, "c"), + }, + .{ + .operation = .insert, + .text = try alloc.dupe(u8, "d"), + }, + .{ + .operation = .equal, + .text = try alloc.dupe(u8, "e"), + }, + .{ + .operation = .equal, + .text = try alloc.dupe(u8, "f"), + }, + }); + try diffCleanupMerge(alloc, &diffs5); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + .{ .operation = .delete, .text = "ac" }, + .{ .operation = .insert, .text = "bd" }, + .{ .operation = .equal, .text = "ef" }, + }), diffs5.items); // Merge interweave // // try diffs2.appendSlice(alloc, &[_]Diff{ // .{ .operation = .delete, .text = "a" }, From 18b75311d281146a1becf905c9bea8a2890a8bbf Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 18:21:30 -0400 Subject: [PATCH 012/176] Free two more clobbered diff texts --- DiffMatchPatch.zig | 130 ++++++++++++++++++++++++++++++--------------- 1 file changed, 87 insertions(+), 43 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 5d764e7..73f02d5 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -918,6 +918,10 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo diffs.items[pointer - 1].text, diffs.items[pointer + 1].text, }); + const old_pt = diffs.items[pointer].text; + defer allocator.free(old_pt); + const old_pt1t = diffs.items[pointer + 1].text; + defer allocator.free(old_pt1t); diffs.items[pointer].text = pt; diffs.items[pointer + 1].text = p1t; freeRangeDiffList(allocator, diffs, pointer - 1, 1); @@ -932,6 +936,10 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo diffs.items[pointer].text[diffs.items[pointer + 1].text.len..], diffs.items[pointer + 1].text, }); + const old_ptm1 = diffs.items[pointer - 1].text; + defer allocator.free(old_ptm1); + const old_pt = diffs.items[pointer].text; + defer allocator.free(old_pt); diffs.items[pointer - 1].text = pm1t; diffs.items[pointer].text = pt; freeRangeDiffList(allocator, diffs, pointer + 1, 1); @@ -1780,49 +1788,85 @@ test diffCleanupMerge { .{ .operation = .insert, .text = "bd" }, .{ .operation = .equal, .text = "ef" }, }), diffs5.items); // Merge interweave - // - // try diffs2.appendSlice(alloc, &[_]Diff{ - // .{ .operation = .delete, .text = "a" }, - // .{ .operation = .insert, .text = "abc" }, - // .{ .operation = .delete, .text = "dc" }, - // }); - // try diffCleanupMerge(alloc, &diffs2); - // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - // .{ .operation = .equal, .text = "a" }, - // .{ .operation = .delete, .text = "d" }, - // .{ .operation = .insert, .text = "b" }, - // .{ .operation = .equal, .text = "c" }, - // }), diffs2.items); // Prefix and suffix detection - // - // diffs2.items.len = 0; - // - // try diffs2.appendSlice(alloc, &[_]Diff{ - // .{ .operation = .equal, .text = "x" }, - // .{ .operation = .delete, .text = "a" }, - // .{ .operation = .insert, .text = "abc" }, - // .{ .operation = .delete, .text = "dc" }, - // .{ .operation = .equal, .text = "y" }, - // }); - // try diffCleanupMerge(alloc, &diffs2); - // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - // .{ .operation = .equal, .text = "xa" }, - // .{ .operation = .delete, .text = "d" }, - // .{ .operation = .insert, .text = "b" }, - // .{ .operation = .equal, .text = "cy" }, - // }), diffs2.items); // Prefix and suffix detection with equalities - // - // diffs2.items.len = 0; - // - // try diffs2.appendSlice(alloc, &[_]Diff{ - // .{ .operation = .equal, .text = "a" }, - // .{ .operation = .insert, .text = "ba" }, - // .{ .operation = .equal, .text = "c" }, - // }); - // try diffCleanupMerge(alloc, &diffs2); - // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - // .{ .operation = .insert, .text = "ab" }, - // .{ .operation = .equal, .text = "ac" }, - // }), diffs2.items); // Slide edit left + + var diffs6 = DiffList{}; + defer deinitDiffList(alloc, &diffs6); + try diffs6.appendSlice(alloc, &[_]Diff{ + .{ + .operation = .delete, + .text = try alloc.dupe(u8, "a"), + }, + .{ + .operation = .insert, + .text = try alloc.dupe(u8, "abc"), + }, + .{ + .operation = .delete, + .text = try alloc.dupe(u8, "dc"), + }, + }); + try diffCleanupMerge(alloc, &diffs6); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + .{ .operation = .equal, .text = "a" }, + .{ .operation = .delete, .text = "d" }, + .{ .operation = .insert, .text = "b" }, + .{ .operation = .equal, .text = "c" }, + }), diffs6.items); // Prefix and suffix detection + + var diffs7 = DiffList{}; + defer deinitDiffList(alloc, &diffs7); + + try diffs7.appendSlice(alloc, &[_]Diff{ + .{ + .operation = .equal, + .text = try alloc.dupe(u8, "x"), + }, + .{ + .operation = .delete, + .text = try alloc.dupe(u8, "a"), + }, + .{ + .operation = .insert, + .text = try alloc.dupe(u8, "abc"), + }, + .{ + .operation = .delete, + .text = try alloc.dupe(u8, "dc"), + }, + .{ + .operation = .equal, + .text = try alloc.dupe(u8, "y"), + }, + }); + try diffCleanupMerge(alloc, &diffs7); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + .{ .operation = .equal, .text = "xa" }, + .{ .operation = .delete, .text = "d" }, + .{ .operation = .insert, .text = "b" }, + .{ .operation = .equal, .text = "cy" }, + }), diffs7.items); // Prefix and suffix detection with equalities + + var diffs8 = DiffList{}; + defer deinitDiffList(alloc, &diffs8); + try diffs8.appendSlice(alloc, &[_]Diff{ + .{ + .operation = .equal, + .text = try alloc.dupe(u8, "a"), + }, + .{ + .operation = .insert, + .text = try alloc.dupe(u8, "ba"), + }, + .{ + .operation = .equal, + .text = try alloc.dupe(u8, "c"), + }, + }); + try diffCleanupMerge(alloc, &diffs8); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + .{ .operation = .insert, .text = "ab" }, + .{ .operation = .equal, .text = "ac" }, + }), diffs8.items); // Slide edit left // // diffs2.items.len = 0; // From 1ebee826e957823d5b91e222fec0d198339702e8 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 18:40:41 -0400 Subject: [PATCH 013/176] Restore all diffCleanMerge tests --- DiffMatchPatch.zig | 186 ++++++++++++++++++++++++++++----------------- 1 file changed, 117 insertions(+), 69 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 73f02d5..6c93bfb 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1867,75 +1867,123 @@ test diffCleanupMerge { .{ .operation = .insert, .text = "ab" }, .{ .operation = .equal, .text = "ac" }, }), diffs8.items); // Slide edit left - // - // diffs2.items.len = 0; - // - // try diffs2.appendSlice(alloc, &[_]Diff{ - // .{ .operation = .equal, .text = "c" }, - // .{ .operation = .insert, .text = "ab" }, - // .{ .operation = .equal, .text = "a" }, - // }); - // try diffCleanupMerge(alloc, &diffs2); - // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - // .{ .operation = .equal, .text = "ca" }, - // .{ .operation = .insert, .text = "ba" }, - // }), diffs2.items); // Slide edit right - // - // diffs2.items.len = 0; - // - // try diffs2.appendSlice(alloc, &[_]Diff{ - // Diff.init(.equal, "a"), - // Diff.init(.delete, "b"), - // Diff.init(.equal, "c"), - // Diff.init(.delete, "ac"), - // Diff.init(.equal, "x"), - // }); - // try diffCleanupMerge(alloc, &diffs2); - // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - // Diff.init(.delete, "abc"), - // Diff.init(.equal, "acx"), - // }), diffs2.items); // Slide edit left recursive - // - // diffs2.items.len = 0; - // - // try diffs2.appendSlice(alloc, &[_]Diff{ - // Diff.init(.equal, "x"), - // Diff.init(.delete, "ca"), - // Diff.init(.equal, "c"), - // Diff.init(.delete, "b"), - // Diff.init(.equal, "a"), - // }); - // try diffCleanupMerge(alloc, &diffs2); - // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - // Diff.init(.equal, "xca"), - // Diff.init(.delete, "cba"), - // }), diffs2.items); // Slide edit right recursive - // - // diffs2.items.len = 0; - // - // try diffs2.appendSlice(alloc, &[_]Diff{ - // Diff.init(.delete, "b"), - // Diff.init(.insert, "ab"), - // Diff.init(.equal, "c"), - // }); - // try diffCleanupMerge(alloc, &diffs2); - // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - // Diff.init(.insert, "a"), - // Diff.init(.equal, "bc"), - // }), diffs2.items); // Empty merge - // - // diffs2.items.len = 0; - // - // try diffs2.appendSlice(alloc, &[_]Diff{ - // Diff.init(.equal, ""), - // Diff.init(.insert, "a"), - // Diff.init(.equal, "b"), - // }); - // try diffCleanupMerge(alloc, &diffs2); - // try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - // Diff.init(.insert, "a"), - // Diff.init(.equal, "b"), - // }), diffs2.items); // Empty equality + + var diffs9 = DiffList{}; + defer deinitDiffList(alloc, &diffs9); + try diffs9.appendSlice(alloc, &[_]Diff{ + .{ + .operation = .equal, + .text = try alloc.dupe(u8, "c"), + }, + .{ + .operation = .insert, + .text = try alloc.dupe(u8, "ab"), + }, + .{ + .operation = .equal, + .text = try alloc.dupe(u8, "a"), + }, + }); + try diffCleanupMerge(alloc, &diffs9); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + .{ .operation = .equal, .text = "ca" }, + .{ .operation = .insert, .text = "ba" }, + }), diffs9.items); // Slide edit right + + var diffs10 = DiffList{}; + defer deinitDiffList(alloc, &diffs10); + try diffs10.appendSlice(alloc, &[_]Diff{ + Diff.init( + .equal, + try alloc.dupe(u8, "a"), + ), + Diff.init( + .delete, + try alloc.dupe(u8, "b"), + ), + Diff.init( + .equal, + try alloc.dupe(u8, "c"), + ), + Diff.init( + .delete, + try alloc.dupe(u8, "ac"), + ), + Diff.init( + .equal, + try alloc.dupe(u8, "x"), + ), + }); + try diffCleanupMerge(alloc, &diffs10); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + Diff.init(.delete, "abc"), + Diff.init(.equal, "acx"), + }), diffs10.items); // Slide edit left recursive + + var diffs11 = DiffList{}; + defer deinitDiffList(alloc, &diffs11); + try diffs11.appendSlice(alloc, &[_]Diff{ + Diff.init( + .equal, + try alloc.dupe(u8, "x"), + ), + Diff.init( + .delete, + try alloc.dupe(u8, "ca"), + ), + Diff.init( + .equal, + try alloc.dupe(u8, "c"), + ), + Diff.init( + .delete, + try alloc.dupe(u8, "b"), + ), + Diff.init( + .equal, + try alloc.dupe(u8, "a"), + ), + }); + try diffCleanupMerge(alloc, &diffs11); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + Diff.init(.equal, "xca"), + Diff.init(.delete, "cba"), + }), diffs11.items); // Slide edit right recursive + + var diffs12 = DiffList{}; + defer deinitDiffList(alloc, &diffs12); + try diffs12.appendSlice(alloc, &[_]Diff{ + Diff.init( + .delete, + try alloc.dupe(u8, "b"), + ), + Diff.init( + .insert, + try alloc.dupe(u8, "ab"), + ), + Diff.init( + .equal, + try alloc.dupe(u8, "c"), + ), + }); + try diffCleanupMerge(alloc, &diffs12); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + Diff.init(.insert, "a"), + Diff.init(.equal, "bc"), + }), diffs12.items); // Empty merge + + var diffs13 = DiffList{}; + defer deinitDiffList(alloc, &diffs13); + try diffs13.appendSlice(alloc, &[_]Diff{ + Diff.init(.equal, ""), + Diff.init(.insert, try alloc.dupe(u8, "a")), + Diff.init(.equal, try alloc.dupe(u8, "b")), + }); + try diffCleanupMerge(alloc, &diffs13); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ + Diff.init(.insert, "a"), + Diff.init(.equal, "b"), + }), diffs13.items); // Empty equality } test diffCleanupSemanticLossless { From 350a85e4817624d219eb4536f75bdc88bd7e9cd7 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 18:50:36 -0400 Subject: [PATCH 014/176] Restore frees to diffCleanupSemanticLossless --- DiffMatchPatch.zig | 77 ++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 6c93bfb..a3ea385 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1176,23 +1176,21 @@ pub fn diffCleanupSemanticLossless( if (!std.mem.eql(u8, diffs.items[pointer - 1].text, best_equality_1.items)) { // We have an improvement, save it back to the diff. if (best_equality_1.items.len != 0) { - // allocator.free(diffs.items[pointer - 1].text); + allocator.free(diffs.items[pointer - 1].text); diffs.items[pointer - 1].text = try allocator.dupe(u8, best_equality_1.items); } else { const old_diff = diffs.orderedRemove(pointer - 1); - // allocator.free(old_diff.text); - _ = old_diff; + allocator.free(old_diff.text); pointer -= 1; } - // allocator.free(diffs.items[pointer].text); + allocator.free(diffs.items[pointer].text); diffs.items[pointer].text = try allocator.dupe(u8, best_edit.items); if (best_equality_2.items.len != 0) { - // allocator.free(diffs.items[pointer - 1].text); + allocator.free(diffs.items[pointer + 1].text); diffs.items[pointer + 1].text = try allocator.dupe(u8, best_equality_2.items); } else { const old_diff = diffs.orderedRemove(pointer + 1); - // allocator.free(old_diff.text); - _ = old_diff; + allocator.free(old_diff.text); pointer -= 1; } } @@ -1987,36 +1985,35 @@ test diffCleanupMerge { } test diffCleanupSemanticLossless { - if (false) { - var arena = std.heap.ArenaAllocator.init(testing.allocator); - defer arena.deinit(); - - var diffs = DiffList{}; - try diffCleanupSemanticLossless(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[0]Diff{}), diffs.items); // Null case - - diffs.items.len = 0; + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.equal, "AAA\r\n\r\nBBB"), - Diff.init(.insert, "\r\nDDD\r\n\r\nBBB"), - Diff.init(.equal, "\r\nEEE"), - }); - try diffCleanupSemanticLossless(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "AAA\r\n\r\n"), - Diff.init(.insert, "BBB\r\nDDD\r\n\r\n"), - Diff.init(.equal, "BBB\r\nEEE"), - }), diffs.items); + const alloc = std.testing.allocator; + var diffs = DiffList{}; + try diffCleanupSemanticLossless(alloc, &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[0]Diff{}), diffs.items); // Null case - diffs.items.len = 0; + var diffs2 = DiffList{}; + defer deinitDiffList(alloc, &diffs2); + try diffs2.appendSlice(alloc, &.{ + Diff.init(.equal, try alloc.dupe(u8, "AAA\r\n\r\nBBB")), + Diff.init(.insert, try alloc.dupe(u8, "\r\nDDD\r\n\r\nBBB")), + Diff.init(.equal, try alloc.dupe(u8, "\r\nEEE")), + }); + try diffCleanupSemanticLossless(alloc, &diffs2); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "AAA\r\n\r\n"), + Diff.init(.insert, "BBB\r\nDDD\r\n\r\n"), + Diff.init(.equal, "BBB\r\nEEE"), + }), diffs2.items); - try diffs.appendSlice(arena.allocator(), &.{ + if (false) { + try diffs.appendSlice(alloc, &.{ Diff.init(.equal, "AAA\r\nBBB"), Diff.init(.insert, " DDD\r\nBBB"), Diff.init(.equal, " EEE"), }); - try diffCleanupSemanticLossless(arena.allocator(), &diffs); + try diffCleanupSemanticLossless(alloc, &diffs); try testing.expectEqualDeep(@as([]const Diff, &.{ Diff.init(.equal, "AAA\r\n"), Diff.init(.insert, "BBB DDD\r\n"), @@ -2025,12 +2022,12 @@ test diffCleanupSemanticLossless { diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ + try diffs.appendSlice(alloc, &.{ Diff.init(.equal, "The c"), Diff.init(.insert, "ow and the c"), Diff.init(.equal, "at."), }); - try diffCleanupSemanticLossless(arena.allocator(), &diffs); + try diffCleanupSemanticLossless(alloc, &diffs); try testing.expectEqualDeep(@as([]const Diff, &.{ Diff.init(.equal, "The "), Diff.init(.insert, "cow and the "), @@ -2039,12 +2036,12 @@ test diffCleanupSemanticLossless { diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ + try diffs.appendSlice(alloc, &.{ Diff.init(.equal, "The-c"), Diff.init(.insert, "ow-and-the-c"), Diff.init(.equal, "at."), }); - try diffCleanupSemanticLossless(arena.allocator(), &diffs); + try diffCleanupSemanticLossless(alloc, &diffs); try testing.expectEqualDeep(@as([]const Diff, &.{ Diff.init(.equal, "The-"), Diff.init(.insert, "cow-and-the-"), @@ -2053,12 +2050,12 @@ test diffCleanupSemanticLossless { diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ + try diffs.appendSlice(alloc, &.{ Diff.init(.equal, "a"), Diff.init(.delete, "a"), Diff.init(.equal, "ax"), }); - try diffCleanupSemanticLossless(arena.allocator(), &diffs); + try diffCleanupSemanticLossless(alloc, &diffs); try testing.expectEqualDeep(@as([]const Diff, &.{ Diff.init(.delete, "a"), Diff.init(.equal, "aax"), @@ -2066,12 +2063,12 @@ test diffCleanupSemanticLossless { diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ + try diffs.appendSlice(alloc, &.{ Diff.init(.equal, "xa"), Diff.init(.delete, "a"), Diff.init(.equal, "a"), }); - try diffCleanupSemanticLossless(arena.allocator(), &diffs); + try diffCleanupSemanticLossless(alloc, &diffs); try testing.expectEqualDeep(@as([]const Diff, &.{ Diff.init(.equal, "xaa"), Diff.init(.delete, "a"), @@ -2079,12 +2076,12 @@ test diffCleanupSemanticLossless { diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ + try diffs.appendSlice(alloc, &.{ Diff.init(.equal, "The xxx. The "), Diff.init(.insert, "zzz. The "), Diff.init(.equal, "yyy."), }); - try diffCleanupSemanticLossless(arena.allocator(), &diffs); + try diffCleanupSemanticLossless(alloc, &diffs); try testing.expectEqualDeep(@as([]const Diff, &.{ Diff.init(.equal, "The xxx."), Diff.init(.insert, " The zzz."), From fe21b517ae642b47e55e8173f88fdb9e993c00fb Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 18:59:18 -0400 Subject: [PATCH 015/176] Tests of diffCleanupSemanticLossless pass --- DiffMatchPatch.zig | 152 ++++++++++++++++++++++----------------------- 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index a3ea385..5b46824 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2007,87 +2007,87 @@ test diffCleanupSemanticLossless { Diff.init(.equal, "BBB\r\nEEE"), }), diffs2.items); - if (false) { - try diffs.appendSlice(alloc, &.{ - Diff.init(.equal, "AAA\r\nBBB"), - Diff.init(.insert, " DDD\r\nBBB"), - Diff.init(.equal, " EEE"), - }); - try diffCleanupSemanticLossless(alloc, &diffs); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "AAA\r\n"), - Diff.init(.insert, "BBB DDD\r\n"), - Diff.init(.equal, "BBB EEE"), - }), diffs.items); - - diffs.items.len = 0; - - try diffs.appendSlice(alloc, &.{ - Diff.init(.equal, "The c"), - Diff.init(.insert, "ow and the c"), - Diff.init(.equal, "at."), - }); - try diffCleanupSemanticLossless(alloc, &diffs); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "The "), - Diff.init(.insert, "cow and the "), - Diff.init(.equal, "cat."), - }), diffs.items); - - diffs.items.len = 0; - - try diffs.appendSlice(alloc, &.{ - Diff.init(.equal, "The-c"), - Diff.init(.insert, "ow-and-the-c"), - Diff.init(.equal, "at."), - }); - try diffCleanupSemanticLossless(alloc, &diffs); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "The-"), - Diff.init(.insert, "cow-and-the-"), - Diff.init(.equal, "cat."), - }), diffs.items); - - diffs.items.len = 0; + var diffs3 = DiffList{}; + defer deinitDiffList(alloc, &diffs3); + try diffs3.appendSlice(alloc, &.{ + Diff.init(.equal, try alloc.dupe(u8, "AAA\r\nBBB")), + Diff.init(.insert, try alloc.dupe(u8, " DDD\r\nBBB")), + Diff.init(.equal, try alloc.dupe(u8, " EEE")), + }); + try diffCleanupSemanticLossless(alloc, &diffs3); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "AAA\r\n"), + Diff.init(.insert, "BBB DDD\r\n"), + Diff.init(.equal, "BBB EEE"), + }), diffs3.items); - try diffs.appendSlice(alloc, &.{ - Diff.init(.equal, "a"), - Diff.init(.delete, "a"), - Diff.init(.equal, "ax"), - }); - try diffCleanupSemanticLossless(alloc, &diffs); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.delete, "a"), - Diff.init(.equal, "aax"), - }), diffs.items); + var diffs4 = DiffList{}; + defer deinitDiffList(alloc, &diffs4); + try diffs4.appendSlice(alloc, &.{ + Diff.init(.equal, try alloc.dupe(u8, "The c")), + Diff.init(.insert, try alloc.dupe(u8, "ow and the c")), + Diff.init(.equal, try alloc.dupe(u8, "at.")), + }); + try diffCleanupSemanticLossless(alloc, &diffs4); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "The "), + Diff.init(.insert, "cow and the "), + Diff.init(.equal, "cat."), + }), diffs4.items); - diffs.items.len = 0; + var diffs5 = DiffList{}; + defer deinitDiffList(alloc, &diffs5); + try diffs5.appendSlice(alloc, &.{ + Diff.init(.equal, try alloc.dupe(u8, "The-c")), + Diff.init(.insert, try alloc.dupe(u8, "ow-and-the-c")), + Diff.init(.equal, try alloc.dupe(u8, "at.")), + }); + try diffCleanupSemanticLossless(alloc, &diffs5); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "The-"), + Diff.init(.insert, "cow-and-the-"), + Diff.init(.equal, "cat."), + }), diffs5.items); - try diffs.appendSlice(alloc, &.{ - Diff.init(.equal, "xa"), - Diff.init(.delete, "a"), - Diff.init(.equal, "a"), - }); - try diffCleanupSemanticLossless(alloc, &diffs); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "xaa"), - Diff.init(.delete, "a"), - }), diffs.items); + var diffs6 = DiffList{}; + defer deinitDiffList(alloc, &diffs6); + try diffs6.appendSlice(alloc, &.{ + Diff.init(.equal, try alloc.dupe(u8, "a")), + Diff.init(.delete, try alloc.dupe(u8, "a")), + Diff.init(.equal, try alloc.dupe(u8, "ax")), + }); + try diffCleanupSemanticLossless(alloc, &diffs6); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.delete, "a"), + Diff.init(.equal, "aax"), + }), diffs6.items); - diffs.items.len = 0; + var diffs7 = DiffList{}; + defer deinitDiffList(alloc, &diffs7); + try diffs7.appendSlice(alloc, &.{ + Diff.init(.equal, try alloc.dupe(u8, "xa")), + Diff.init(.delete, try alloc.dupe(u8, "a")), + Diff.init(.equal, try alloc.dupe(u8, "a")), + }); + try diffCleanupSemanticLossless(alloc, &diffs7); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "xaa"), + Diff.init(.delete, "a"), + }), diffs7.items); - try diffs.appendSlice(alloc, &.{ - Diff.init(.equal, "The xxx. The "), - Diff.init(.insert, "zzz. The "), - Diff.init(.equal, "yyy."), - }); - try diffCleanupSemanticLossless(alloc, &diffs); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "The xxx."), - Diff.init(.insert, " The zzz."), - Diff.init(.equal, " The yyy."), - }), diffs.items); - } + var diffs8 = DiffList{}; + defer deinitDiffList(alloc, &diffs8); + try diffs8.appendSlice(alloc, &.{ + Diff.init(.equal, try alloc.dupe(u8, "The xxx. The ")), + Diff.init(.insert, try alloc.dupe(u8, "zzz. The ")), + Diff.init(.equal, try alloc.dupe(u8, "yyy.")), + }); + try diffCleanupSemanticLossless(alloc, &diffs8); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "The xxx."), + Diff.init(.insert, " The zzz."), + Diff.init(.equal, " The yyy."), + }), diffs8.items); } fn rebuildtexts(allocator: std.mem.Allocator, diffs: DiffList) ![2][]const u8 { From 5ff7a7648462c501fa78469a89c1998008ffede0 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 19:04:10 -0400 Subject: [PATCH 016/176] alloc -> allocator --- DiffMatchPatch.zig | 268 ++++++++++++++++++++++----------------------- 1 file changed, 131 insertions(+), 137 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 5b46824..a4ed93b 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1662,125 +1662,122 @@ test diffCharsToLines { } test diffCleanupMerge { - var arena = std.heap.ArenaAllocator.init(testing.allocator); - defer arena.deinit(); - - const alloc = std.testing.allocator; + const allocator = std.testing.allocator; // Cleanup a messy diff. var diffs = DiffList{}; - defer deinitDiffList(alloc, &diffs); + defer deinitDiffList(allocator, &diffs); try testing.expectEqualDeep(@as([]const Diff, &[0]Diff{}), diffs.items); // Null case - try diffs.appendSlice(alloc, &[_]Diff{ + try diffs.appendSlice(allocator, &[_]Diff{ .{ .operation = .equal, - .text = try alloc.dupe(u8, "a"), + .text = try allocator.dupe(u8, "a"), }, .{ .operation = .delete, - .text = try alloc.dupe(u8, "b"), + .text = try allocator.dupe(u8, "b"), }, .{ .operation = .insert, - .text = try alloc.dupe(u8, "c"), + .text = try allocator.dupe(u8, "c"), }, }); - try diffCleanupMerge(alloc, &diffs); + try diffCleanupMerge(allocator, &diffs); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "b" }, .{ .operation = .insert, .text = "c" } }), diffs.items); // No change case var diffs2 = DiffList{}; - defer deinitDiffList(alloc, &diffs2); - try diffs2.appendSlice(alloc, &[_]Diff{ + defer deinitDiffList(allocator, &diffs2); + try diffs2.appendSlice(allocator, &[_]Diff{ .{ .operation = .equal, - .text = try alloc.dupe(u8, "a"), + .text = try allocator.dupe(u8, "a"), }, .{ .operation = .equal, - .text = try alloc.dupe(u8, "b"), + .text = try allocator.dupe(u8, "b"), }, .{ .operation = .equal, - .text = try alloc.dupe(u8, "c"), + .text = try allocator.dupe(u8, "c"), }, }); - try diffCleanupMerge(alloc, &diffs2); + try diffCleanupMerge(allocator, &diffs2); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .equal, .text = "abc" }, }), diffs2.items); // Merge equalities var diffs3 = DiffList{}; - defer deinitDiffList(alloc, &diffs3); + defer deinitDiffList(allocator, &diffs3); - try diffs3.appendSlice(alloc, &[_]Diff{ + try diffs3.appendSlice(allocator, &[_]Diff{ .{ .operation = .delete, - .text = try alloc.dupe(u8, "a"), + .text = try allocator.dupe(u8, "a"), }, .{ .operation = .delete, - .text = try alloc.dupe(u8, "b"), + .text = try allocator.dupe(u8, "b"), }, .{ .operation = .delete, - .text = try alloc.dupe(u8, "c"), + .text = try allocator.dupe(u8, "c"), }, }); - try diffCleanupMerge(alloc, &diffs3); + try diffCleanupMerge(allocator, &diffs3); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .delete, .text = "abc" }, }), diffs3.items); // Merge deletions var diffs4 = DiffList{}; - defer deinitDiffList(alloc, &diffs4); - try diffs4.appendSlice(alloc, &[_]Diff{ + defer deinitDiffList(allocator, &diffs4); + try diffs4.appendSlice(allocator, &[_]Diff{ .{ .operation = .insert, - .text = try alloc.dupe(u8, "a"), + .text = try allocator.dupe(u8, "a"), }, .{ .operation = .insert, - .text = try alloc.dupe(u8, "b"), + .text = try allocator.dupe(u8, "b"), }, .{ .operation = .insert, - .text = try alloc.dupe(u8, "c"), + .text = try allocator.dupe(u8, "c"), }, }); - try diffCleanupMerge(alloc, &diffs4); + try diffCleanupMerge(allocator, &diffs4); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .insert, .text = "abc" }, }), diffs4.items); // Merge insertions var diffs5 = DiffList{}; - defer deinitDiffList(alloc, &diffs5); - try diffs5.appendSlice(alloc, &[_]Diff{ + defer deinitDiffList(allocator, &diffs5); + try diffs5.appendSlice(allocator, &[_]Diff{ .{ .operation = .delete, - .text = try alloc.dupe(u8, "a"), + .text = try allocator.dupe(u8, "a"), }, .{ .operation = .insert, - .text = try alloc.dupe(u8, "b"), + .text = try allocator.dupe(u8, "b"), }, .{ .operation = .delete, - .text = try alloc.dupe(u8, "c"), + .text = try allocator.dupe(u8, "c"), }, .{ .operation = .insert, - .text = try alloc.dupe(u8, "d"), + .text = try allocator.dupe(u8, "d"), }, .{ .operation = .equal, - .text = try alloc.dupe(u8, "e"), + .text = try allocator.dupe(u8, "e"), }, .{ .operation = .equal, - .text = try alloc.dupe(u8, "f"), + .text = try allocator.dupe(u8, "f"), }, }); - try diffCleanupMerge(alloc, &diffs5); + try diffCleanupMerge(allocator, &diffs5); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .delete, .text = "ac" }, .{ .operation = .insert, .text = "bd" }, @@ -1788,22 +1785,22 @@ test diffCleanupMerge { }), diffs5.items); // Merge interweave var diffs6 = DiffList{}; - defer deinitDiffList(alloc, &diffs6); - try diffs6.appendSlice(alloc, &[_]Diff{ + defer deinitDiffList(allocator, &diffs6); + try diffs6.appendSlice(allocator, &[_]Diff{ .{ .operation = .delete, - .text = try alloc.dupe(u8, "a"), + .text = try allocator.dupe(u8, "a"), }, .{ .operation = .insert, - .text = try alloc.dupe(u8, "abc"), + .text = try allocator.dupe(u8, "abc"), }, .{ .operation = .delete, - .text = try alloc.dupe(u8, "dc"), + .text = try allocator.dupe(u8, "dc"), }, }); - try diffCleanupMerge(alloc, &diffs6); + try diffCleanupMerge(allocator, &diffs6); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "d" }, @@ -1812,31 +1809,31 @@ test diffCleanupMerge { }), diffs6.items); // Prefix and suffix detection var diffs7 = DiffList{}; - defer deinitDiffList(alloc, &diffs7); + defer deinitDiffList(allocator, &diffs7); - try diffs7.appendSlice(alloc, &[_]Diff{ + try diffs7.appendSlice(allocator, &[_]Diff{ .{ .operation = .equal, - .text = try alloc.dupe(u8, "x"), + .text = try allocator.dupe(u8, "x"), }, .{ .operation = .delete, - .text = try alloc.dupe(u8, "a"), + .text = try allocator.dupe(u8, "a"), }, .{ .operation = .insert, - .text = try alloc.dupe(u8, "abc"), + .text = try allocator.dupe(u8, "abc"), }, .{ .operation = .delete, - .text = try alloc.dupe(u8, "dc"), + .text = try allocator.dupe(u8, "dc"), }, .{ .operation = .equal, - .text = try alloc.dupe(u8, "y"), + .text = try allocator.dupe(u8, "y"), }, }); - try diffCleanupMerge(alloc, &diffs7); + try diffCleanupMerge(allocator, &diffs7); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .equal, .text = "xa" }, .{ .operation = .delete, .text = "d" }, @@ -1845,139 +1842,139 @@ test diffCleanupMerge { }), diffs7.items); // Prefix and suffix detection with equalities var diffs8 = DiffList{}; - defer deinitDiffList(alloc, &diffs8); - try diffs8.appendSlice(alloc, &[_]Diff{ + defer deinitDiffList(allocator, &diffs8); + try diffs8.appendSlice(allocator, &[_]Diff{ .{ .operation = .equal, - .text = try alloc.dupe(u8, "a"), + .text = try allocator.dupe(u8, "a"), }, .{ .operation = .insert, - .text = try alloc.dupe(u8, "ba"), + .text = try allocator.dupe(u8, "ba"), }, .{ .operation = .equal, - .text = try alloc.dupe(u8, "c"), + .text = try allocator.dupe(u8, "c"), }, }); - try diffCleanupMerge(alloc, &diffs8); + try diffCleanupMerge(allocator, &diffs8); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .insert, .text = "ab" }, .{ .operation = .equal, .text = "ac" }, }), diffs8.items); // Slide edit left var diffs9 = DiffList{}; - defer deinitDiffList(alloc, &diffs9); - try diffs9.appendSlice(alloc, &[_]Diff{ + defer deinitDiffList(allocator, &diffs9); + try diffs9.appendSlice(allocator, &[_]Diff{ .{ .operation = .equal, - .text = try alloc.dupe(u8, "c"), + .text = try allocator.dupe(u8, "c"), }, .{ .operation = .insert, - .text = try alloc.dupe(u8, "ab"), + .text = try allocator.dupe(u8, "ab"), }, .{ .operation = .equal, - .text = try alloc.dupe(u8, "a"), + .text = try allocator.dupe(u8, "a"), }, }); - try diffCleanupMerge(alloc, &diffs9); + try diffCleanupMerge(allocator, &diffs9); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .equal, .text = "ca" }, .{ .operation = .insert, .text = "ba" }, }), diffs9.items); // Slide edit right var diffs10 = DiffList{}; - defer deinitDiffList(alloc, &diffs10); - try diffs10.appendSlice(alloc, &[_]Diff{ + defer deinitDiffList(allocator, &diffs10); + try diffs10.appendSlice(allocator, &[_]Diff{ Diff.init( .equal, - try alloc.dupe(u8, "a"), + try allocator.dupe(u8, "a"), ), Diff.init( .delete, - try alloc.dupe(u8, "b"), + try allocator.dupe(u8, "b"), ), Diff.init( .equal, - try alloc.dupe(u8, "c"), + try allocator.dupe(u8, "c"), ), Diff.init( .delete, - try alloc.dupe(u8, "ac"), + try allocator.dupe(u8, "ac"), ), Diff.init( .equal, - try alloc.dupe(u8, "x"), + try allocator.dupe(u8, "x"), ), }); - try diffCleanupMerge(alloc, &diffs10); + try diffCleanupMerge(allocator, &diffs10); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ Diff.init(.delete, "abc"), Diff.init(.equal, "acx"), }), diffs10.items); // Slide edit left recursive var diffs11 = DiffList{}; - defer deinitDiffList(alloc, &diffs11); - try diffs11.appendSlice(alloc, &[_]Diff{ + defer deinitDiffList(allocator, &diffs11); + try diffs11.appendSlice(allocator, &[_]Diff{ Diff.init( .equal, - try alloc.dupe(u8, "x"), + try allocator.dupe(u8, "x"), ), Diff.init( .delete, - try alloc.dupe(u8, "ca"), + try allocator.dupe(u8, "ca"), ), Diff.init( .equal, - try alloc.dupe(u8, "c"), + try allocator.dupe(u8, "c"), ), Diff.init( .delete, - try alloc.dupe(u8, "b"), + try allocator.dupe(u8, "b"), ), Diff.init( .equal, - try alloc.dupe(u8, "a"), + try allocator.dupe(u8, "a"), ), }); - try diffCleanupMerge(alloc, &diffs11); + try diffCleanupMerge(allocator, &diffs11); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ Diff.init(.equal, "xca"), Diff.init(.delete, "cba"), }), diffs11.items); // Slide edit right recursive var diffs12 = DiffList{}; - defer deinitDiffList(alloc, &diffs12); - try diffs12.appendSlice(alloc, &[_]Diff{ + defer deinitDiffList(allocator, &diffs12); + try diffs12.appendSlice(allocator, &[_]Diff{ Diff.init( .delete, - try alloc.dupe(u8, "b"), + try allocator.dupe(u8, "b"), ), Diff.init( .insert, - try alloc.dupe(u8, "ab"), + try allocator.dupe(u8, "ab"), ), Diff.init( .equal, - try alloc.dupe(u8, "c"), + try allocator.dupe(u8, "c"), ), }); - try diffCleanupMerge(alloc, &diffs12); + try diffCleanupMerge(allocator, &diffs12); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ Diff.init(.insert, "a"), Diff.init(.equal, "bc"), }), diffs12.items); // Empty merge var diffs13 = DiffList{}; - defer deinitDiffList(alloc, &diffs13); - try diffs13.appendSlice(alloc, &[_]Diff{ + defer deinitDiffList(allocator, &diffs13); + try diffs13.appendSlice(allocator, &[_]Diff{ Diff.init(.equal, ""), - Diff.init(.insert, try alloc.dupe(u8, "a")), - Diff.init(.equal, try alloc.dupe(u8, "b")), + Diff.init(.insert, try allocator.dupe(u8, "a")), + Diff.init(.equal, try allocator.dupe(u8, "b")), }); - try diffCleanupMerge(alloc, &diffs13); + try diffCleanupMerge(allocator, &diffs13); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ Diff.init(.insert, "a"), Diff.init(.equal, "b"), @@ -1985,22 +1982,19 @@ test diffCleanupMerge { } test diffCleanupSemanticLossless { - var arena = std.heap.ArenaAllocator.init(testing.allocator); - defer arena.deinit(); - - const alloc = std.testing.allocator; + const allocator = std.testing.allocator; var diffs = DiffList{}; - try diffCleanupSemanticLossless(alloc, &diffs); + try diffCleanupSemanticLossless(allocator, &diffs); try testing.expectEqualDeep(@as([]const Diff, &[0]Diff{}), diffs.items); // Null case var diffs2 = DiffList{}; - defer deinitDiffList(alloc, &diffs2); - try diffs2.appendSlice(alloc, &.{ - Diff.init(.equal, try alloc.dupe(u8, "AAA\r\n\r\nBBB")), - Diff.init(.insert, try alloc.dupe(u8, "\r\nDDD\r\n\r\nBBB")), - Diff.init(.equal, try alloc.dupe(u8, "\r\nEEE")), + defer deinitDiffList(allocator, &diffs2); + try diffs2.appendSlice(allocator, &.{ + Diff.init(.equal, try allocator.dupe(u8, "AAA\r\n\r\nBBB")), + Diff.init(.insert, try allocator.dupe(u8, "\r\nDDD\r\n\r\nBBB")), + Diff.init(.equal, try allocator.dupe(u8, "\r\nEEE")), }); - try diffCleanupSemanticLossless(alloc, &diffs2); + try diffCleanupSemanticLossless(allocator, &diffs2); try testing.expectEqualDeep(@as([]const Diff, &.{ Diff.init(.equal, "AAA\r\n\r\n"), Diff.init(.insert, "BBB\r\nDDD\r\n\r\n"), @@ -2008,13 +2002,13 @@ test diffCleanupSemanticLossless { }), diffs2.items); var diffs3 = DiffList{}; - defer deinitDiffList(alloc, &diffs3); - try diffs3.appendSlice(alloc, &.{ - Diff.init(.equal, try alloc.dupe(u8, "AAA\r\nBBB")), - Diff.init(.insert, try alloc.dupe(u8, " DDD\r\nBBB")), - Diff.init(.equal, try alloc.dupe(u8, " EEE")), + defer deinitDiffList(allocator, &diffs3); + try diffs3.appendSlice(allocator, &.{ + Diff.init(.equal, try allocator.dupe(u8, "AAA\r\nBBB")), + Diff.init(.insert, try allocator.dupe(u8, " DDD\r\nBBB")), + Diff.init(.equal, try allocator.dupe(u8, " EEE")), }); - try diffCleanupSemanticLossless(alloc, &diffs3); + try diffCleanupSemanticLossless(allocator, &diffs3); try testing.expectEqualDeep(@as([]const Diff, &.{ Diff.init(.equal, "AAA\r\n"), Diff.init(.insert, "BBB DDD\r\n"), @@ -2022,13 +2016,13 @@ test diffCleanupSemanticLossless { }), diffs3.items); var diffs4 = DiffList{}; - defer deinitDiffList(alloc, &diffs4); - try diffs4.appendSlice(alloc, &.{ - Diff.init(.equal, try alloc.dupe(u8, "The c")), - Diff.init(.insert, try alloc.dupe(u8, "ow and the c")), - Diff.init(.equal, try alloc.dupe(u8, "at.")), + defer deinitDiffList(allocator, &diffs4); + try diffs4.appendSlice(allocator, &.{ + Diff.init(.equal, try allocator.dupe(u8, "The c")), + Diff.init(.insert, try allocator.dupe(u8, "ow and the c")), + Diff.init(.equal, try allocator.dupe(u8, "at.")), }); - try diffCleanupSemanticLossless(alloc, &diffs4); + try diffCleanupSemanticLossless(allocator, &diffs4); try testing.expectEqualDeep(@as([]const Diff, &.{ Diff.init(.equal, "The "), Diff.init(.insert, "cow and the "), @@ -2036,13 +2030,13 @@ test diffCleanupSemanticLossless { }), diffs4.items); var diffs5 = DiffList{}; - defer deinitDiffList(alloc, &diffs5); - try diffs5.appendSlice(alloc, &.{ - Diff.init(.equal, try alloc.dupe(u8, "The-c")), - Diff.init(.insert, try alloc.dupe(u8, "ow-and-the-c")), - Diff.init(.equal, try alloc.dupe(u8, "at.")), + defer deinitDiffList(allocator, &diffs5); + try diffs5.appendSlice(allocator, &.{ + Diff.init(.equal, try allocator.dupe(u8, "The-c")), + Diff.init(.insert, try allocator.dupe(u8, "ow-and-the-c")), + Diff.init(.equal, try allocator.dupe(u8, "at.")), }); - try diffCleanupSemanticLossless(alloc, &diffs5); + try diffCleanupSemanticLossless(allocator, &diffs5); try testing.expectEqualDeep(@as([]const Diff, &.{ Diff.init(.equal, "The-"), Diff.init(.insert, "cow-and-the-"), @@ -2050,39 +2044,39 @@ test diffCleanupSemanticLossless { }), diffs5.items); var diffs6 = DiffList{}; - defer deinitDiffList(alloc, &diffs6); - try diffs6.appendSlice(alloc, &.{ - Diff.init(.equal, try alloc.dupe(u8, "a")), - Diff.init(.delete, try alloc.dupe(u8, "a")), - Diff.init(.equal, try alloc.dupe(u8, "ax")), + defer deinitDiffList(allocator, &diffs6); + try diffs6.appendSlice(allocator, &.{ + Diff.init(.equal, try allocator.dupe(u8, "a")), + Diff.init(.delete, try allocator.dupe(u8, "a")), + Diff.init(.equal, try allocator.dupe(u8, "ax")), }); - try diffCleanupSemanticLossless(alloc, &diffs6); + try diffCleanupSemanticLossless(allocator, &diffs6); try testing.expectEqualDeep(@as([]const Diff, &.{ Diff.init(.delete, "a"), Diff.init(.equal, "aax"), }), diffs6.items); var diffs7 = DiffList{}; - defer deinitDiffList(alloc, &diffs7); - try diffs7.appendSlice(alloc, &.{ - Diff.init(.equal, try alloc.dupe(u8, "xa")), - Diff.init(.delete, try alloc.dupe(u8, "a")), - Diff.init(.equal, try alloc.dupe(u8, "a")), + defer deinitDiffList(allocator, &diffs7); + try diffs7.appendSlice(allocator, &.{ + Diff.init(.equal, try allocator.dupe(u8, "xa")), + Diff.init(.delete, try allocator.dupe(u8, "a")), + Diff.init(.equal, try allocator.dupe(u8, "a")), }); - try diffCleanupSemanticLossless(alloc, &diffs7); + try diffCleanupSemanticLossless(allocator, &diffs7); try testing.expectEqualDeep(@as([]const Diff, &.{ Diff.init(.equal, "xaa"), Diff.init(.delete, "a"), }), diffs7.items); var diffs8 = DiffList{}; - defer deinitDiffList(alloc, &diffs8); - try diffs8.appendSlice(alloc, &.{ - Diff.init(.equal, try alloc.dupe(u8, "The xxx. The ")), - Diff.init(.insert, try alloc.dupe(u8, "zzz. The ")), - Diff.init(.equal, try alloc.dupe(u8, "yyy.")), + defer deinitDiffList(allocator, &diffs8); + try diffs8.appendSlice(allocator, &.{ + Diff.init(.equal, try allocator.dupe(u8, "The xxx. The ")), + Diff.init(.insert, try allocator.dupe(u8, "zzz. The ")), + Diff.init(.equal, try allocator.dupe(u8, "yyy.")), }); - try diffCleanupSemanticLossless(alloc, &diffs8); + try diffCleanupSemanticLossless(allocator, &diffs8); try testing.expectEqualDeep(@as([]const Diff, &.{ Diff.init(.equal, "The xxx."), Diff.init(.insert, " The zzz."), From b56449f455d374caefe254a9a23a49099f620589 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 19:33:15 -0400 Subject: [PATCH 017/176] Tests pass for diffBisect --- DiffMatchPatch.zig | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index a4ed93b..646ef65 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -119,8 +119,8 @@ fn diffInternal( deadline: u64, ) DiffError!DiffList { // Check for equality (speedup). - var diffs = DiffList{}; if (std.mem.eql(u8, before, after)) { + var diffs = DiffList{}; if (before.len != 0) { try diffs.append(allocator, Diff.init(.equal, try allocator.dupe(u8, before))); } @@ -140,7 +140,7 @@ fn diffInternal( trimmed_after = trimmed_after[0 .. trimmed_after.len - common_length]; // Compute the diff on the middle block. - diffs = try dmp.diffCompute(allocator, trimmed_before, trimmed_after, check_lines, deadline); + var diffs = try dmp.diffCompute(allocator, trimmed_before, trimmed_after, check_lines, deadline); // Restore the prefix and suffix. if (common_prefix.len != 0) { @@ -2105,9 +2105,7 @@ fn rebuildtexts(allocator: std.mem.Allocator, diffs: DiffList) ![2][]const u8 { } test diffBisect { - var arena = std.heap.ArenaAllocator.init(talloc); - defer arena.deinit(); - + const allocator = std.testing.allocator; // Normal. const a = "cat"; const b = "map"; @@ -2115,26 +2113,36 @@ test diffBisect { // the insertion and deletion pairs are swapped. // If the order changes, tweak this test as required. var diffs = DiffList{}; - defer diffs.deinit(arena.allocator()); + defer deinitDiffList(allocator, &diffs); var this = default; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "c"), - Diff.init(.insert, "m"), - Diff.init(.equal, "a"), - Diff.init(.delete, "t"), - Diff.init(.insert, "p"), + try diffs.appendSlice(allocator, &.{ + Diff.init(.delete, try allocator.dupe(u8, "c")), + Diff.init(.insert, try allocator.dupe(u8, "m")), + Diff.init(.equal, try allocator.dupe(u8, "a")), + Diff.init(.delete, try allocator.dupe(u8, "t")), + Diff.init(.insert, try allocator.dupe(u8, "p")), }); // Travis TODO not sure if maxInt(u64) is correct for DateTime.MaxValue - try testing.expectEqualDeep(diffs, try this.diffBisect(arena.allocator(), a, b, std.math.maxInt(u64))); // Normal. + var diff_bisect = try this.diffBisect( + allocator, + a, + b, + std.math.maxInt(u64), + ); + defer deinitDiffList(allocator, &diff_bisect); + try testing.expectEqualDeep(diffs, diff_bisect); // Normal. // Timeout. - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "cat"), - Diff.init(.insert, "map"), + var diffs2 = DiffList{}; + defer deinitDiffList(allocator, &diffs2); + try diffs2.appendSlice(allocator, &.{ + Diff.init(.delete, try allocator.dupe(u8, "cat")), + Diff.init(.insert, try allocator.dupe(u8, "map")), }); // Travis TODO not sure if 0 is correct for DateTime.MinValue - try testing.expectEqualDeep(diffs, try this.diffBisect(arena.allocator(), a, b, 0)); // Timeout. + var diff_bisect2 = try this.diffBisect(allocator, a, b, 0); + defer deinitDiffList(allocator, &diff_bisect2); + try testing.expectEqualDeep(diffs2, diff_bisect2); // Timeout. } const talloc = testing.allocator; From 2388fff63442371aa9e4aa68ccf332eba7ea4e67 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 20:36:39 -0400 Subject: [PATCH 018/176] Catch a few leaks, lather, rinse, repeat --- DiffMatchPatch.zig | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 646ef65..1078bbb 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -645,6 +645,12 @@ fn diffLineMode( if (count_delete >= 1 and count_insert >= 1) { // Delete the offending records and add the merged ones. // diffs.RemoveRange(pointer - count_delete - count_insert, count_delete + count_insert); + freeRangeDiffList( + allocator, + &diffs, + pointer - count_delete - count_insert, + count_delete + count_insert, + ); try diffs.replaceRange( allocator, pointer - count_delete - count_insert, @@ -652,7 +658,8 @@ fn diffLineMode( &.{}, ); pointer = pointer - count_delete - count_insert; - const sub_diff = try dmp.diffInternal(allocator, text_delete.items, text_insert.items, false, deadline); + var sub_diff = try dmp.diffInternal(allocator, text_delete.items, text_insert.items, false, deadline); + defer sub_diff.deinit(allocator); // diffs.InsertRange(pointer, sub_diff); try diffs.insertSlice(allocator, pointer, sub_diff.items); pointer = pointer + sub_diff.items.len; @@ -1048,10 +1055,10 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError @intCast(pointer), Diff.init(.equal, try allocator.dupe(u8, insertion[0..overlap_length1])), ); - // XXX activate: allocator.free(diffs.items[@inteCast(pointer-1)].text); + allocator.free(diffs.items[@intCast(pointer - 1)].text); diffs.items[@intCast(pointer - 1)].text = try allocator.dupe(u8, deletion[0 .. deletion.len - overlap_length1]); - // XXX activate: allocator.free(diffs.items[@inteCast(pointer+1)].text); + allocator.free(diffs.items[@intCast(pointer + 1)].text); diffs.items[@intCast(pointer + 1)].text = try allocator.dupe(u8, insertion[overlap_length1..]); pointer += 1; @@ -1069,11 +1076,11 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError ); diffs.items[@intCast(pointer - 1)].operation = .insert; const new_minus = try allocator.dupe(u8, insertion[0 .. insertion.len - overlap_length2]); - // XXX activate: allocator.free(diffs.items[@inteCast(pointer-1)].text); + allocator.free(diffs.items[@intCast(pointer - 1)].text); diffs.items[@intCast(pointer - 1)].text = new_minus; diffs.items[@intCast(pointer + 1)].operation = .delete; const new_plus = try allocator.dupe(u8, deletion[overlap_length2..]); - // XXX activate: allocator.free(diffs.items[@inteCast(pointer+1)].text); + allocator.free(diffs.items[@intCast(pointer + 1)].text); diffs.items[@intCast(pointer + 1)].text = new_plus; pointer += 1; } @@ -2146,9 +2153,12 @@ test diffBisect { } const talloc = testing.allocator; -test diff { + +// XXX rename to diff +test "diff main" { var arena = std.heap.ArenaAllocator.init(talloc); defer arena.deinit(); + const alloc = std.testing.allocator; // Perform a trivial diff. var diffs = DiffList{}; @@ -2213,7 +2223,8 @@ test diff { const a = "`Twas brillig, and the slithy toves\nDid gyre and gimble in the wabe:\nAll mimsy were the borogoves,\nAnd the mome raths outgrabe.\n" ** 1024; const b = "I am the very model of a modern major general,\nI've information vegetable, animal, and mineral,\nI know the kings of England, and I quote the fights historical,\nFrom Marathon to Waterloo, in order categorical.\n" ** 1024; const start_time = std.time.milliTimestamp(); - _ = try this.diff(arena.allocator(), a, b, false); // Travis - TODO not sure what the third arg should be + var time_diff = try this.diff(alloc, a, b, false); // Travis - TODO not sure what the third arg should be + defer deinitDiffList(alloc, &time_diff); const end_time = std.time.milliTimestamp(); // Test that we took at least the timeout period. try testing.expect(this.diff_timeout <= end_time - start_time); // diff: Timeout min. @@ -2228,7 +2239,11 @@ test diff { // Must be long to pass the 100 char cutoff. const a = "1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n"; const b = "abcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\n"; - try testing.expectEqualDeep(try this.diff(arena.allocator(), a, b, true), try this.diff(arena.allocator(), a, b, false)); // diff: Simple line-mode. + var diff_checked = try this.diff(alloc, a, b, true); + defer deinitDiffList(alloc, &diff_checked); + var diff_unchecked = try this.diff(alloc, a, b, false); + defer deinitDiffList(alloc, &diff_unchecked); + try testing.expectEqualDeep(diff_checked, diff_unchecked); // diff: Simple line-mode. } { const a = "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"; From 7a47bd7d7384b0cc1132356b7e68641383af3235 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 21:29:45 -0400 Subject: [PATCH 019/176] Defer free until slice is taken I had missed that some slices in cleanupSemantic were being taken from the same string/slice that was already at that index, due to the prior code renaming them. Ganbatte! --- DiffMatchPatch.zig | 260 ++++++++++++++++++++++++--------------------- 1 file changed, 137 insertions(+), 123 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 1078bbb..3609788 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -969,6 +969,7 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError var changes = false; // Stack of indices where equalities are found. var equalities = ArrayListUnmanaged(isize){}; + defer equalities.deinit(allocator); // Always equal to equalities[equalitiesLength-1][1] var last_equality: ?[]const u8 = null; var pointer: isize = 0; // Index of current position. @@ -1050,15 +1051,15 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError { // Overlap found. // Insert an equality and trim the surrounding edits. + defer allocator.free(deletion); + defer allocator.free(insertion); try diffs.insert( allocator, @intCast(pointer), Diff.init(.equal, try allocator.dupe(u8, insertion[0..overlap_length1])), ); - allocator.free(diffs.items[@intCast(pointer - 1)].text); diffs.items[@intCast(pointer - 1)].text = try allocator.dupe(u8, deletion[0 .. deletion.len - overlap_length1]); - allocator.free(diffs.items[@intCast(pointer + 1)].text); diffs.items[@intCast(pointer + 1)].text = try allocator.dupe(u8, insertion[overlap_length1..]); pointer += 1; @@ -1069,6 +1070,8 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError { // Reverse overlap found. // Insert an equality and swap and trim the surrounding edits. + defer allocator.free(deletion); + defer allocator.free(insertion); try diffs.insert( allocator, @intCast(pointer), @@ -1076,7 +1079,6 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError ); diffs.items[@intCast(pointer - 1)].operation = .insert; const new_minus = try allocator.dupe(u8, insertion[0 .. insertion.len - overlap_length2]); - allocator.free(diffs.items[@intCast(pointer - 1)].text); diffs.items[@intCast(pointer - 1)].text = new_minus; diffs.items[@intCast(pointer + 1)].operation = .delete; const new_plus = try allocator.dupe(u8, deletion[overlap_length2..]); @@ -2248,7 +2250,11 @@ test "diff main" { { const a = "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"; const b = "abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij"; - try testing.expectEqualDeep(try this.diff(arena.allocator(), a, b, true), try this.diff(arena.allocator(), a, b, false)); // diff: Single line-mode. + var diff_checked = try this.diff(alloc, a, b, true); + defer deinitDiffList(alloc, &diff_checked); + var diff_unchecked = try this.diff(alloc, a, b, false); + defer deinitDiffList(alloc, &diff_unchecked); + try testing.expectEqualDeep(diff_checked, diff_unchecked); // diff: Single line-mode. } const a = "1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n"; @@ -2264,139 +2270,147 @@ test "diff main" { arena.allocator().free(texts_textmode[1]); } try testing.expectEqualDeep(texts_textmode, texts_linemode); // diff: Overlap line-mode. - - // Test null inputs -- not needed because nulls can't be passed in C#. } test diffCleanupSemantic { - if (false) { - var arena = std.heap.ArenaAllocator.init(talloc); - defer arena.deinit(); + var arena = std.heap.ArenaAllocator.init(talloc); + defer arena.deinit(); - // Cleanup semantically trivial equalities. - // Null case. - var diffs = DiffList{}; - defer diffs.deinit(arena.allocator()); - // var this = default; - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqual(@as(usize, 0), diffs.items.len); // Null case + const alloc = std.testing.allocator; + // Cleanup semantically trivial equalities. + // Null case. + var diffs_empty = DiffList{}; + defer deinitDiffList(alloc, &diffs_empty); + // var this = default; + try diffCleanupSemantic(arena.allocator(), &diffs_empty); + try testing.expectEqual(@as(usize, 0), diffs_empty.items.len); // Null case - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "ab"), - Diff.init(.insert, "cd"), - Diff.init(.equal, "12"), - Diff.init(.delete, "e"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // No elimination #1 - Diff.init(.delete, "ab"), - Diff.init(.insert, "cd"), - Diff.init(.equal, "12"), - Diff.init(.delete, "e"), - }), diffs.items); + var diffs = DiffList{}; + defer deinitDiffList(alloc, &diffs); + diffs.items.len = 0; + try diffs.appendSlice(alloc, &.{ + Diff.init(.delete, try alloc.dupe(u8, "ab")), + Diff.init(.insert, try alloc.dupe(u8, "cd")), + Diff.init(.equal, try alloc.dupe(u8, "12")), + Diff.init(.delete, try alloc.dupe(u8, "e")), + }); + try diffCleanupSemantic(alloc, &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // No elimination #1 + Diff.init(.delete, "ab"), + Diff.init(.insert, "cd"), + Diff.init(.equal, "12"), + Diff.init(.delete, "e"), + }), diffs.items); - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "abc"), - Diff.init(.insert, "ABC"), - Diff.init(.equal, "1234"), - Diff.init(.delete, "wxyz"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // No elimination #2 - Diff.init(.delete, "abc"), - Diff.init(.insert, "ABC"), - Diff.init(.equal, "1234"), - Diff.init(.delete, "wxyz"), - }), diffs.items); + var diffs2 = DiffList{}; + defer deinitDiffList(alloc, &diffs2); + diffs2.items.len = 0; + try diffs2.appendSlice(alloc, &.{ + Diff.init(.delete, try alloc.dupe(u8, "abc")), + Diff.init(.insert, try alloc.dupe(u8, "ABC")), + Diff.init(.equal, try alloc.dupe(u8, "1234")), + Diff.init(.delete, try alloc.dupe(u8, "wxyz")), + }); + try diffCleanupSemantic(alloc, &diffs2); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // No elimination #2 + Diff.init(.delete, "abc"), + Diff.init(.insert, "ABC"), + Diff.init(.equal, "1234"), + Diff.init(.delete, "wxyz"), + }), diffs2.items); - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "a"), - Diff.init(.equal, "b"), - Diff.init(.delete, "c"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Simple elimination - Diff.init(.delete, "abc"), - Diff.init(.insert, "b"), - }), diffs.items); + var diffs3 = DiffList{}; + defer deinitDiffList(alloc, &diffs3); + try diffs3.appendSlice(alloc, &.{ + Diff.init(.delete, try alloc.dupe(u8, "a")), + Diff.init(.equal, try alloc.dupe(u8, "b")), + Diff.init(.delete, try alloc.dupe(u8, "c")), + }); + try diffCleanupSemantic(alloc, &diffs3); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Simple elimination + Diff.init(.delete, "abc"), + Diff.init(.insert, "b"), + }), diffs3.items); - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "ab"), - Diff.init(.equal, "cd"), - Diff.init(.delete, "e"), - Diff.init(.equal, "f"), - Diff.init(.insert, "g"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Backpass elimination - Diff.init(.delete, "abcdef"), - Diff.init(.insert, "cdfg"), - }), diffs.items); + var diffs4 = DiffList{}; + defer deinitDiffList(alloc, &diffs4); + try diffs4.appendSlice(alloc, &.{ + Diff.init(.delete, try alloc.dupe(u8, "ab")), + Diff.init(.equal, try alloc.dupe(u8, "cd")), + Diff.init(.delete, try alloc.dupe(u8, "e")), + Diff.init(.equal, try alloc.dupe(u8, "f")), + Diff.init(.insert, try alloc.dupe(u8, "g")), + }); + try diffCleanupSemantic(alloc, &diffs4); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Backpass elimination + Diff.init(.delete, "abcdef"), + Diff.init(.insert, "cdfg"), + }), diffs4.items); - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.insert, "1"), - Diff.init(.equal, "A"), - Diff.init(.delete, "B"), - Diff.init(.insert, "2"), - Diff.init(.equal, "_"), - Diff.init(.insert, "1"), - Diff.init(.equal, "A"), - Diff.init(.delete, "B"), - Diff.init(.insert, "2"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Multiple elimination - Diff.init(.delete, "AB_AB"), - Diff.init(.insert, "1A2_1A2"), - }), diffs.items); + var diffs5 = DiffList{}; + defer deinitDiffList(alloc, &diffs5); + try diffs5.appendSlice(alloc, &.{ + Diff.init(.insert, try alloc.dupe(u8, "1")), + Diff.init(.equal, try alloc.dupe(u8, "A")), + Diff.init(.delete, try alloc.dupe(u8, "B")), + Diff.init(.insert, try alloc.dupe(u8, "2")), + Diff.init(.equal, try alloc.dupe(u8, "_")), + Diff.init(.insert, try alloc.dupe(u8, "1")), + Diff.init(.equal, try alloc.dupe(u8, "A")), + Diff.init(.delete, try alloc.dupe(u8, "B")), + Diff.init(.insert, try alloc.dupe(u8, "2")), + }); + try diffCleanupSemantic(alloc, &diffs5); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Multiple elimination + Diff.init(.delete, "AB_AB"), + Diff.init(.insert, "1A2_1A2"), + }), diffs5.items); - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.equal, "The c"), - Diff.init(.delete, "ow and the c"), - Diff.init(.equal, "at."), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Word boundaries - Diff.init(.equal, "The "), - Diff.init(.delete, "cow and the "), - Diff.init(.equal, "cat."), - }), diffs.items); + var diffs6 = DiffList{}; + defer deinitDiffList(alloc, &diffs6); + try diffs6.appendSlice(alloc, &.{ + Diff.init(.equal, try alloc.dupe(u8, "The c")), + Diff.init(.delete, try alloc.dupe(u8, "ow and the c")), + Diff.init(.equal, try alloc.dupe(u8, "at.")), + }); + try diffCleanupSemantic(alloc, &diffs6); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Word boundaries + Diff.init(.equal, "The "), + Diff.init(.delete, "cow and the "), + Diff.init(.equal, "cat."), + }), diffs6.items); - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "abcxx"), - Diff.init(.insert, "xxdef"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // No overlap elimination - Diff.init(.delete, "abcxx"), - Diff.init(.insert, "xxdef"), - }), diffs.items); + var diffs7 = DiffList{}; + defer deinitDiffList(alloc, &diffs7); + try diffs7.appendSlice(alloc, &.{ + Diff.init(.delete, try alloc.dupe(u8, "abcxx")), + Diff.init(.insert, try alloc.dupe(u8, "xxdef")), + }); + try diffCleanupSemantic(alloc, &diffs7); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // No overlap elimination + Diff.init(.delete, "abcxx"), + Diff.init(.insert, "xxdef"), + }), diffs7.items); - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ - Diff.init(.delete, "abcxxx"), - Diff.init(.insert, "xxxdef"), - }); - try diffCleanupSemantic(arena.allocator(), &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Overlap elimination - Diff.init(.delete, "abc"), - Diff.init(.equal, "xxx"), - Diff.init(.insert, "def"), - }), diffs.items); + var diffs8 = DiffList{}; + defer deinitDiffList(alloc, &diffs8); + try diffs8.appendSlice(alloc, &.{ + Diff.init(.delete, try alloc.dupe(u8, "abcxxx")), + Diff.init(.insert, try alloc.dupe(u8, "xxxdef")), + }); + try diffCleanupSemantic(alloc, &diffs8); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Overlap elimination + Diff.init(.delete, "abc"), + Diff.init(.equal, "xxx"), + Diff.init(.insert, "def"), + }), diffs8.items); - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ + if (false) { + try diffs.appendSlice(alloc, &.{ Diff.init(.delete, "xxxabc"), Diff.init(.insert, "defxxx"), }); - try diffCleanupSemantic(arena.allocator(), &diffs); + try diffCleanupSemantic(alloc, &diffs); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Reverse overlap elimination Diff.init(.insert, "def"), Diff.init(.equal, "xxx"), @@ -2404,14 +2418,14 @@ test diffCleanupSemantic { }), diffs.items); diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ + try diffs.appendSlice(alloc, &.{ Diff.init(.delete, "abcd1212"), Diff.init(.insert, "1212efghi"), Diff.init(.equal, "----"), Diff.init(.delete, "A3"), Diff.init(.insert, "3BC"), }); - try diffCleanupSemantic(arena.allocator(), &diffs); + try diffCleanupSemantic(alloc, &diffs); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Two overlap eliminations Diff.init(.delete, "abcd"), Diff.init(.equal, "1212"), From eecdb470e552ad9152c5231c548e4fd492877918 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 2 Jul 2024 23:05:05 -0400 Subject: [PATCH 020/176] Fait accompli The library now manages its own memory, as it should. --- DiffMatchPatch.zig | 202 +++++++++++++++++++-------------------------- 1 file changed, 86 insertions(+), 116 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 3609788..6945b71 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -6,7 +6,9 @@ const Allocator = std.mem.Allocator; const ArrayListUnmanaged = std.ArrayListUnmanaged; const DiffList = ArrayListUnmanaged(Diff); -fn deinitDiffList(allocator: Allocator, diffs: *DiffList) void { +/// Deinit an `ArrayListUnmanaged(Diff)` and the allocated slices of +/// text in each `Diff`. +pub fn deinitDiffList(allocator: Allocator, diffs: *DiffList) void { defer diffs.deinit(allocator); for (diffs.items) |d| { if (d.text.len > 0) { @@ -84,8 +86,6 @@ patch_margin: u16 = 4, pub const DiffError = error{OutOfMemory}; -/// It is recommended that you use an Arena for this operation. -/// /// Find the differences between two texts. /// @param before Old string to be diffed. /// @param after New string to be diffed. @@ -711,7 +711,6 @@ fn diffLinesToChars( // "\x00" is a valid character, but various debuggers don't like it. // So we'll insert a junk entry to avoid generating a null character. - // XXX why is this necessary? -Sam try line_array.append(allocator, ""); // Allocate 2/3rds of the space for text1, the rest for text2. @@ -736,7 +735,7 @@ fn diffLinesToCharsMunge( ) DiffError![]const u8 { var line_start: isize = 0; var line_end: isize = -1; - var line: []const u8 = ""; + var line: []const u8 = undefined; var chars = ArrayListUnmanaged(u8){}; defer chars.deinit(allocator); // Walk the text, pulling out a Substring for each line. @@ -1082,7 +1081,6 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError diffs.items[@intCast(pointer - 1)].text = new_minus; diffs.items[@intCast(pointer + 1)].operation = .delete; const new_plus = try allocator.dupe(u8, deletion[overlap_length2..]); - allocator.free(diffs.items[@intCast(pointer + 1)].text); diffs.items[@intCast(pointer + 1)].text = new_plus; pointer += 1; } @@ -1262,16 +1260,6 @@ fn diffCleanupSemanticScore(one: []const u8, two: []const u8) usize { return 0; } -// Define some regex patterns for matching boundaries. -// private Regex BLANKLINEEND = new Regex("\\n\\r?\\n\\Z"); -// \n\n -// \n\r\n -// private Regex BLANKLINESTART = new Regex("\\A\\r?\\n\\r?\\n"); -// \n\n -// \r\n\n -// \n\r\n -// \r\n\r\n - /// Reduce the number of edits by eliminating operationally trivial /// equalities. pub fn diffCleanupEfficiency( @@ -1408,36 +1396,10 @@ fn diffCommonOverlap(text1_in: []const u8, text2_in: []const u8) usize { } } -// pub fn main() void { -// var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); -// defer arena.deinit(); - -// var bruh = default.diff(arena.allocator(), "Hello World.", "Goodbye World.", true); -// std.log.err("{any}", .{bruh}); -// } - -// test { -// var arena = std.heap.ArenaAllocator.init(testing.allocator); -// defer arena.deinit(); - -// var bruh = try default.diff(arena.allocator(), "Hello World.", "Goodbye World.", true); -// try diffCleanupSemantic(arena.allocator(), &bruh); -// for (bruh.items) |b| { -// std.log.err("{any}", .{b}); -// } - -// // for (bruh.items) |b| { -// // std.log.err("{s} {s}", .{ switch (b.operation) { -// // .equal => "", -// // .insert => "+", -// // .delete => "-", -// // }, b.text }); -// // } -// } - -// TODO: Allocate all text in diffs to +// DONE [✅]: Allocate all text in diffs to // not cause segfault while freeing; not a problem -// at the moment because we don't free anything :P +// at the moment because we don't free anything :( +// (or was it??) test diffCommonPrefix { // Detect any common suffix. @@ -1609,27 +1571,36 @@ test diffLinesToChars { // TODO: More than 256 to reveal any 8-bit limitations but this requires // some unicode logic that I don't want to deal with + // + // Casting to Unicode is straightforward and should sort correctly, I'm + // more concerned about the weird behavior when the 'char' is equal to a + // newline. Uncomment the EqualSlices below to see what I mean. + // I think there's some cleanup logic in the actual linediff that should + // take care of the problem, but I don't like it. + + const n: u8 = 255; + tmp_array_list.items.len = 0; - // TODO: Fix this - - // const n: u8 = 255; - // tmp_array_list.items.len = 0; - - // var line_list = std.ArrayList(u8).init(alloc); - // var char_list = std.ArrayList(u8).init(alloc); - - // var i: u8 = 0; - // while (i < n) : (i += 1) { - // try tmp_array_list.append(&.{ i, '\n' }); - // try line_list.appendSlice(&.{ i, '\n' }); - // try char_list.append(i); - // } - // try testing.expectEqual(@as(usize, n), tmp_array_list.items.len); // Test initialization fail #1 - // try testing.expectEqual(@as(usize, n), char_list.items.len); // Test initialization fail #2 - // try tmp_array_list.insert(0, ""); - // result = try diffLinesToChars(alloc, line_list.items, ""); - // try testing.expectEqualStrings(char_list.items, result.chars_1); - // try testing.expectEqualStrings("", result.chars_2); + var line_list = std.ArrayList(u8).init(allocator); + defer line_list.deinit(); + var char_list = std.ArrayList(u8).init(allocator); + defer char_list.deinit(); + + var i: u8 = 1; + while (i < n) : (i += 1) { + try tmp_array_list.append(&.{ i, '\n' }); + try line_list.appendSlice(&.{ i, '\n' }); + try char_list.append(i); + } + try testing.expectEqual(@as(usize, n - 1), tmp_array_list.items.len); // Test initialization fail #1 + try testing.expectEqual(@as(usize, n - 1), char_list.items.len); // Test initialization fail #2 + try tmp_array_list.insert(0, ""); + result = try diffLinesToChars(allocator, line_list.items, ""); + defer result.deinit(allocator); + // TODO: This isn't equal, should it be? + // try testing.expectEqualSlices(u8, char_list.items, result.chars_1); + try testing.expectEqualStrings("", result.chars_2); + // TODO this is wrong because of the max_value I think? // try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); } @@ -1676,7 +1647,10 @@ test diffCleanupMerge { var diffs = DiffList{}; defer deinitDiffList(allocator, &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[0]Diff{}), diffs.items); // Null case + try testing.expectEqualDeep( + @as([]const Diff, &[0]Diff{}), + diffs.items, + ); // Null case try diffs.appendSlice(allocator, &[_]Diff{ .{ @@ -1694,6 +1668,7 @@ test diffCleanupMerge { }); try diffCleanupMerge(allocator, &diffs); try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "b" }, .{ .operation = .insert, .text = "c" } }), diffs.items); // No change case + var diffs2 = DiffList{}; defer deinitDiffList(allocator, &diffs2); try diffs2.appendSlice(allocator, &[_]Diff{ @@ -1717,7 +1692,6 @@ test diffCleanupMerge { var diffs3 = DiffList{}; defer deinitDiffList(allocator, &diffs3); - try diffs3.appendSlice(allocator, &[_]Diff{ .{ .operation = .delete, @@ -1819,7 +1793,6 @@ test diffCleanupMerge { var diffs7 = DiffList{}; defer deinitDiffList(allocator, &diffs7); - try diffs7.appendSlice(allocator, &[_]Diff{ .{ .operation = .equal, @@ -2156,11 +2129,10 @@ test diffBisect { const talloc = testing.allocator; -// XXX rename to diff -test "diff main" { +test diff { var arena = std.heap.ArenaAllocator.init(talloc); defer arena.deinit(); - const alloc = std.testing.allocator; + const allocator = std.testing.allocator; // Perform a trivial diff. var diffs = DiffList{}; @@ -2225,8 +2197,8 @@ test "diff main" { const a = "`Twas brillig, and the slithy toves\nDid gyre and gimble in the wabe:\nAll mimsy were the borogoves,\nAnd the mome raths outgrabe.\n" ** 1024; const b = "I am the very model of a modern major general,\nI've information vegetable, animal, and mineral,\nI know the kings of England, and I quote the fights historical,\nFrom Marathon to Waterloo, in order categorical.\n" ** 1024; const start_time = std.time.milliTimestamp(); - var time_diff = try this.diff(alloc, a, b, false); // Travis - TODO not sure what the third arg should be - defer deinitDiffList(alloc, &time_diff); + var time_diff = try this.diff(allocator, a, b, false); // Travis - TODO not sure what the third arg should be + defer deinitDiffList(allocator, &time_diff); const end_time = std.time.milliTimestamp(); // Test that we took at least the timeout period. try testing.expect(this.diff_timeout <= end_time - start_time); // diff: Timeout min. @@ -2241,19 +2213,19 @@ test "diff main" { // Must be long to pass the 100 char cutoff. const a = "1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n"; const b = "abcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\n"; - var diff_checked = try this.diff(alloc, a, b, true); - defer deinitDiffList(alloc, &diff_checked); - var diff_unchecked = try this.diff(alloc, a, b, false); - defer deinitDiffList(alloc, &diff_unchecked); + var diff_checked = try this.diff(allocator, a, b, true); + defer deinitDiffList(allocator, &diff_checked); + var diff_unchecked = try this.diff(allocator, a, b, false); + defer deinitDiffList(allocator, &diff_unchecked); try testing.expectEqualDeep(diff_checked, diff_unchecked); // diff: Simple line-mode. } { const a = "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"; const b = "abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij"; - var diff_checked = try this.diff(alloc, a, b, true); - defer deinitDiffList(alloc, &diff_checked); - var diff_unchecked = try this.diff(alloc, a, b, false); - defer deinitDiffList(alloc, &diff_unchecked); + var diff_checked = try this.diff(allocator, a, b, true); + defer deinitDiffList(allocator, &diff_checked); + var diff_unchecked = try this.diff(allocator, a, b, false); + defer deinitDiffList(allocator, &diff_unchecked); try testing.expectEqualDeep(diff_checked, diff_unchecked); // diff: Single line-mode. } @@ -2273,16 +2245,13 @@ test "diff main" { } test diffCleanupSemantic { - var arena = std.heap.ArenaAllocator.init(talloc); - defer arena.deinit(); - const alloc = std.testing.allocator; // Cleanup semantically trivial equalities. // Null case. var diffs_empty = DiffList{}; defer deinitDiffList(alloc, &diffs_empty); // var this = default; - try diffCleanupSemantic(arena.allocator(), &diffs_empty); + try diffCleanupSemantic(alloc, &diffs_empty); try testing.expectEqual(@as(usize, 0), diffs_empty.items.len); // Null case var diffs = DiffList{}; @@ -2405,35 +2374,36 @@ test diffCleanupSemantic { Diff.init(.insert, "def"), }), diffs8.items); - if (false) { - try diffs.appendSlice(alloc, &.{ - Diff.init(.delete, "xxxabc"), - Diff.init(.insert, "defxxx"), - }); - try diffCleanupSemantic(alloc, &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Reverse overlap elimination - Diff.init(.insert, "def"), - Diff.init(.equal, "xxx"), - Diff.init(.delete, "abc"), - }), diffs.items); - - diffs.items.len = 0; - try diffs.appendSlice(alloc, &.{ - Diff.init(.delete, "abcd1212"), - Diff.init(.insert, "1212efghi"), - Diff.init(.equal, "----"), - Diff.init(.delete, "A3"), - Diff.init(.insert, "3BC"), - }); - try diffCleanupSemantic(alloc, &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Two overlap eliminations - Diff.init(.delete, "abcd"), - Diff.init(.equal, "1212"), - Diff.init(.insert, "efghi"), - Diff.init(.equal, "----"), - Diff.init(.delete, "A"), - Diff.init(.equal, "3"), - Diff.init(.insert, "BC"), - }), diffs.items); - } + var diffs9 = DiffList{}; + defer deinitDiffList(alloc, &diffs9); + try diffs9.appendSlice(alloc, &.{ + Diff.init(.delete, try alloc.dupe(u8, "xxxabc")), + Diff.init(.insert, try alloc.dupe(u8, "defxxx")), + }); + try diffCleanupSemantic(alloc, &diffs9); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Reverse overlap elimination + Diff.init(.insert, "def"), + Diff.init(.equal, "xxx"), + Diff.init(.delete, "abc"), + }), diffs9.items); + + var diffs10 = DiffList{}; + defer deinitDiffList(alloc, &diffs10); + try diffs10.appendSlice(alloc, &.{ + Diff.init(.delete, try alloc.dupe(u8, "abcd1212")), + Diff.init(.insert, try alloc.dupe(u8, "1212efghi")), + Diff.init(.equal, try alloc.dupe(u8, "----")), + Diff.init(.delete, try alloc.dupe(u8, "A3")), + Diff.init(.insert, try alloc.dupe(u8, "3BC")), + }); + try diffCleanupSemantic(alloc, &diffs10); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Two overlap eliminations + Diff.init(.delete, "abcd"), + Diff.init(.equal, "1212"), + Diff.init(.insert, "efghi"), + Diff.init(.equal, "----"), + Diff.init(.delete, "A"), + Diff.init(.equal, "3"), + Diff.init(.insert, "BC"), + }), diffs10.items); } From 7032b4b8289945a187f53ab6b6025dc551bc0ff4 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 3 Jul 2024 09:29:19 -0400 Subject: [PATCH 021/176] Polish things up --- DiffMatchPatch.zig | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 6945b71..d5a7d9f 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -632,19 +632,16 @@ fn diffLineMode( switch (diffs.items[pointer].operation) { .insert => { count_insert += 1; - // text_insert += diffs.items[pointer].text; try text_insert.appendSlice(allocator, diffs.items[pointer].text); }, .delete => { count_delete += 1; - // text_delete += diffs.items[pointer].text; try text_delete.appendSlice(allocator, diffs.items[pointer].text); }, .equal => { // Upon reaching an equality, check for prior redundancies. if (count_delete >= 1 and count_insert >= 1) { // Delete the offending records and add the merged ones. - // diffs.RemoveRange(pointer - count_delete - count_insert, count_delete + count_insert); freeRangeDiffList( allocator, &diffs, @@ -660,7 +657,6 @@ fn diffLineMode( pointer = pointer - count_delete - count_insert; var sub_diff = try dmp.diffInternal(allocator, text_delete.items, text_insert.items, false, deadline); defer sub_diff.deinit(allocator); - // diffs.InsertRange(pointer, sub_diff); try diffs.insertSlice(allocator, pointer, sub_diff.items); pointer = pointer + sub_diff.items.len; } @@ -672,8 +668,7 @@ fn diffLineMode( } pointer += 1; } - // diffs.RemoveAt(diffs.Count - 1); // Remove the dummy entry at the end. - diffs.items.len -= 1; + diffs.items.len -= 1; // Remove the dummy entry at the end. return diffs; } @@ -2140,6 +2135,8 @@ test diff { var this = DiffMatchPatch{}; try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "", "", false)).items); // diff: Null case. + // TODO This is the last set of tests using the arena. Someone should + // rewrite them not to do so. -Sam diffs.items.len = 0; try diffs.appendSlice(arena.allocator(), &.{Diff.init(.equal, "abc")}); try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "abc", "abc", false)).items); // diff: Equality. @@ -2191,13 +2188,15 @@ test diff { try diffs.appendSlice(arena.allocator(), &.{ Diff.init(.insert, " "), Diff.init(.equal, "a"), Diff.init(.insert, "nd"), Diff.init(.equal, " [[Pennsylvania]]"), Diff.init(.delete, " and [[New") }); try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "a [[Pennsylvania]] and [[New", " and [[Pennsylvania]]", false)).items); // diff: Large equality. + // end of Arena Zone + this.diff_timeout = 100; // 100ms // Increase the text lengths by 1024 times to ensure a timeout. { const a = "`Twas brillig, and the slithy toves\nDid gyre and gimble in the wabe:\nAll mimsy were the borogoves,\nAnd the mome raths outgrabe.\n" ** 1024; const b = "I am the very model of a modern major general,\nI've information vegetable, animal, and mineral,\nI know the kings of England, and I quote the fights historical,\nFrom Marathon to Waterloo, in order categorical.\n" ** 1024; const start_time = std.time.milliTimestamp(); - var time_diff = try this.diff(allocator, a, b, false); // Travis - TODO not sure what the third arg should be + var time_diff = try this.diff(allocator, a, b, false); defer deinitDiffList(allocator, &time_diff); const end_time = std.time.milliTimestamp(); // Test that we took at least the timeout period. @@ -2231,15 +2230,19 @@ test diff { const a = "1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n"; const b = "abcdefghij\n1234567890\n1234567890\n1234567890\nabcdefghij\n1234567890\n1234567890\n1234567890\nabcdefghij\n1234567890\n1234567890\n1234567890\nabcdefghij\n"; - const texts_linemode = try rebuildtexts(arena.allocator(), try this.diff(arena.allocator(), a, b, true)); + var diffs_linemode = try this.diff(allocator, a, b, true); + defer deinitDiffList(allocator, &diffs_linemode); + const texts_linemode = try rebuildtexts(allocator, diffs_linemode); defer { - arena.allocator().free(texts_linemode[0]); - arena.allocator().free(texts_linemode[1]); + allocator.free(texts_linemode[0]); + allocator.free(texts_linemode[1]); } - const texts_textmode = try rebuildtexts(arena.allocator(), try this.diff(arena.allocator(), a, b, false)); + var diffs_textmode = try this.diff(allocator, a, b, false); + defer deinitDiffList(allocator, &diffs_textmode); + const texts_textmode = try rebuildtexts(allocator, diffs_textmode); defer { - arena.allocator().free(texts_textmode[0]); - arena.allocator().free(texts_textmode[1]); + allocator.free(texts_textmode[0]); + allocator.free(texts_textmode[1]); } try testing.expectEqualDeep(texts_textmode, texts_linemode); // diff: Overlap line-mode. } From 87b1b323026cab13b1b3ce3b9b65507ec363ea95 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 3 Jul 2024 10:53:47 -0400 Subject: [PATCH 022/176] Remove unused temp diff --- DiffMatchPatch.zig | 3 --- 1 file changed, 3 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index d5a7d9f..6350a5f 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -255,9 +255,6 @@ fn diffCompute( ); defer diffs_b.deinit(allocator); - var tmp_diffs = diffs; - defer tmp_diffs.deinit(allocator); - // Merge the results. diffs = diffs_a; try diffs.append(allocator, Diff.init(.equal, try allocator.dupe(u8, half_match.common_middle))); From 20190993b330ec54936f229622a0882536edf24e Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 3 Jul 2024 12:15:13 -0400 Subject: [PATCH 023/176] Modifies prefix and suffix diffs to not split UTF-8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A diff splitting a UTF-8 codepoint is possible: Ω and ϩ, for example, share a common suffix. This split will make the diff invalid UTF-8, even if both texts are valid. A greater concern is common prefixes: every Greek letter between Α and ο shares the prefix byte 0xce, meaning that splits on a prefix would be the rule, not the exception, in normal multibyte texts. This patch prevents the prefix and suffix diffing from ever splitting on such common prefixes and suffixes, by backing out when a diff lands on a follow byte. It remains to ensure that other splits are never in the midst of a multibyte codepoint. --- DiffMatchPatch.zig | 72 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 13 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 6350a5f..161430c 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2,6 +2,7 @@ const DiffMatchPatch = @This(); const std = @import("std"); const testing = std.testing; +const assert = std.debug.assert; const Allocator = std.mem.Allocator; const ArrayListUnmanaged = std.ArrayListUnmanaged; const DiffList = ArrayListUnmanaged(Diff); @@ -154,26 +155,71 @@ fn diffInternal( return diffs; } +/// Test if a byte is a UTF-8 follow byte +inline fn is_follow(byte: u8) bool { + return byte & 0b1100_0000 == 0b1000_0000; +} + +/// Find a common prefix which respects UTF-8 code point boundaries. fn diffCommonPrefix(before: []const u8, after: []const u8) usize { const n = @min(before.len, after.len); var i: usize = 0; while (i < n) : (i += 1) { - if (before[i] != after[i]) { - return i; + var b = before[i]; + const a = after[i]; + if (a != b) { + if (is_follow(a) and is_follow(b)) { + // We've clipped a codepoint, back out + if (i == 0) return i; // Malformed UTF-8 is always possible + i -= 1; + // We'll track `before` since they must be the same: + b = before[i]; + assert(b == after[i]); + while (i != 0 and is_follow(b)) { + i -= 1; + b = before[i]; + assert(b == after[i]); + } + // Now we're either at zero, or at the lead: + return i; + } else { + return i; + } } } return n; } +/// Find a common suffix which respects UTF-8 code point boundaries fn diffCommonSuffix(before: []const u8, after: []const u8) usize { const n = @min(before.len, after.len); var i: usize = 1; - + var was_follow = false; while (i <= n) : (i += 1) { - if (before[before.len - i] != after[after.len - i]) { - return i - 1; + var b = before[before.len - i]; + const a = after[after.len - i]; + if (a != b) { + if (was_follow) { + // Means we're at at least 2: + assert(i > 1); + // We just saw an identical follow byte, so we back + // out forward: + i -= 1; + b = before[before.len - i]; + assert(b == after[after.len - i]); + while (i > 1 and is_follow(b)) { + i -= 1; + b = before[before.len - i]; + assert(b == after[after.len - i]); + } // Either at one, or no more follow bytes: + return i - 1; + } else { + return i - 1; + } + } else { + was_follow = is_follow(b); // no need to check twice } } @@ -812,38 +858,38 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo // Upon reaching an equality, check for prior redundancies. if (count_delete + count_insert > 1) { if (count_delete != 0 and count_insert != 0) { - // Factor out any common prefixies. + // Factor out any common prefixes. common_length = diffCommonPrefix(text_insert.items, text_delete.items); if (common_length != 0) { if ((pointer - count_delete - count_insert) > 0 and diffs.items[pointer - count_delete - count_insert - 1].operation == .equal) - { + { // The prefix is not at the start of the diffs const ii = pointer - count_delete - count_insert - 1; var nt = try allocator.alloc(u8, diffs.items[ii].text.len + common_length); - const ot = diffs.items[ii].text; defer allocator.free(ot); @memcpy(nt[0..ot.len], ot); @memcpy(nt[ot.len..], text_insert.items[0..common_length]); diffs.items[ii].text = nt; - } else { + } else { // The prefix is at the start of the diffs const text = try allocator.dupe(u8, text_insert.items[0..common_length]); try diffs.insert(allocator, 0, Diff.init(.equal, text)); - pointer += 1; + pointer += 1; // Keep pointer pointed at current diff } + // Remove merged prefixes try text_insert.replaceRange(allocator, 0, common_length, &.{}); try text_delete.replaceRange(allocator, 0, common_length, &.{}); } // Factor out any common suffixies. - // @ZigPort this seems very wrong common_length = diffCommonSuffix(text_insert.items, text_delete.items); if (common_length != 0) { + // Move the common part to the equal diff const old_text = diffs.items[pointer].text; defer allocator.free(old_text); diffs.items[pointer].text = try std.mem.concat(allocator, u8, &.{ text_insert.items[text_insert.items.len - common_length ..], old_text, - }); + }); // Remove it from the ends of the insert/delete pair text_insert.items.len -= common_length; text_delete.items.len -= common_length; } @@ -872,7 +918,7 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo pointer += 1; } else if (pointer != 0 and diffs.items[pointer - 1].operation == .equal) { // Merge this equality with the previous one. - // TODO: Fix using realloc or smth + // Diff texts are []const u8 so a realloc isn't practical here var nt = try allocator.alloc(u8, diffs.items[pointer - 1].text.len + diffs.items[pointer].text.len); const ot = diffs.items[pointer - 1].text; defer (allocator.free(ot)); From 006d7a1f50668cf0f650fb5e7248dca0b992f2e3 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 3 Jul 2024 12:38:04 -0400 Subject: [PATCH 024/176] Add (failing!) greek test --- DiffMatchPatch.zig | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 161430c..803b64d 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -266,6 +266,7 @@ fn diffCompute( .delete else .insert; + // No need to adjust this index since any split is already valid try diffs.append(allocator, Diff.init(op, try allocator.dupe(u8, long_text[0..index]))); try diffs.append(allocator, Diff.init(.equal, try allocator.dupe(u8, short_text))); try diffs.append(allocator, Diff.init(op, try allocator.dupe(u8, long_text[index + short_text.len ..]))); @@ -2290,6 +2291,23 @@ test diff { try testing.expectEqualDeep(texts_textmode, texts_linemode); // diff: Overlap line-mode. } +test "Unicode diffs" { + const allocator = std.testing.allocator; + const this = DiffMatchPatch{}; + var greek_diff = try this.diff( + allocator, + "αβγ", + "αβδ", + false, + ); + defer deinitDiffList(allocator, &greek_diff); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "αβ"), + Diff.init(.insert, "δ"), + Diff.init(.equal, "γ"), + }), greek_diff.items); +} + test diffCleanupSemantic { const alloc = std.testing.allocator; // Cleanup semantically trivial equalities. From 3cd7a49c75f18856274560a499118771423ba317 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 3 Jul 2024 16:32:42 -0400 Subject: [PATCH 025/176] Add equalForwards and equalBackwards for Myers 1986 The only thing which matters here is not splitting codepoints down the middle. Since prefix and suffix matching will also refuse to split, later repair stages won't re-split on us. As the comments note, this doesn't validate UTF-8, and might create less-than-perfect diffs if the texts contain bad Unicode, a situation and outcome I do not care about even slightly. In addition to fixing the problem where diffs would be invalid utf-8 in many ordinary cases, what this gets us is the ability to use codepoints as indices for the diffLine function. Since diffing won't damage utf-8 sequences, we can diff in linemode and decode each sequence back to the codepoint, which will serve as an offset into the line_array. --- DiffMatchPatch.zig | 142 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 128 insertions(+), 14 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 803b64d..935ece5 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -164,7 +164,6 @@ inline fn is_follow(byte: u8) bool { fn diffCommonPrefix(before: []const u8, after: []const u8) usize { const n = @min(before.len, after.len); var i: usize = 0; - while (i < n) : (i += 1) { var b = before[i]; const a = after[i]; @@ -312,7 +311,6 @@ fn diffCompute( if (check_lines and before.len > 100 and after.len > 100) { return dmp.diffLineMode(allocator, before, after, deadline); } - return dmp.diffBisect(allocator, before, after, deadline); } @@ -523,11 +521,14 @@ fn diffBisect( x1 = v1.items[@intCast(k1_offset - 1)] + 1; } var y1 = x1 - k1; - while (x1 < before_length and - y1 < after_length and before[@intCast(x1)] == after[@intCast(y1)]) - { - x1 += 1; - y1 += 1; + while (x1 < before_length and y1 < after_length) { + const match, const d1 = equalForward(before, after, x1, y1); + if (match) { + x1 += d1; + y1 += d1; + } else { + break; + } } v1.items[@intCast(k1_offset)] = x1; if (x1 > before_length) { @@ -562,12 +563,19 @@ fn diffBisect( x2 = v2.items[@intCast(k2_offset - 1)] + 1; } var y2: isize = x2 - k2; - while (x2 < before_length and y2 < after_length and - before[@intCast(before_length - x2 - 1)] == - after[@intCast(after_length - y2 - 1)]) - { - x2 += 1; - y2 += 1; + while (x2 < before_length and y2 < after_length) { + const match, const d1 = equalBackward( + before, + after, + before_length - x2 - 1, + after_length - y2 - 1, + ); + if (match) { + x2 += d1; + y2 += d1; + } else { + break; + } } v2.items[@intCast(k2_offset)] = x2; if (x2 > before_length) { @@ -599,6 +607,112 @@ fn diffBisect( return diffs; } +/// Match up to a full character in the forward direction. Note the +/// goal here: we aren't validating Unicode, we're making sure we don't +/// split code unit sequences. We might get non-minimal diffs on bad +/// UTF-8, but that's fine. +fn equalForward( + before: []const u8, + after: []const u8, + b_i: isize, + a_i: isize, +) struct { bool, isize } { + const b_u: usize = @intCast(b_i); + const a_u: usize = @intCast(a_i); + const b1c = before[b_u]; + const a1c = after[a_u]; + if (b1c == a1c) { + // how many codeunits might we expect? + // ASCII is easy: + if (b1c < 128) { + return .{ true, 1 }; + } else { + switch (b1c) { + 0xc2...0xdf => { + // two bytes + if (b_u + 1 >= before.len or a_u + 1 >= after.len) { + // it's a match ¯\_(ツ)_/¯ + return .{ true, 1 }; + } // length is unused for false results + return .{ before[b_u + 1] == after[a_u + 1], 2 }; + }, + 0xe0...0xef => { + // three bytes + if (b_u + 2 >= before.len or a_u + 2 >= after.len) { + return .{ true, 1 }; + } + const m2 = before[b_u + 1] == after[a_u + 1]; + const m3 = before[b_u + 2] == after[a_u + 2]; + return .{ m2 and m3, 3 }; + }, + 0xf0...0xf4 => { + // four bytes + if (b_u + 3 >= before.len or a_u + 3 >= after.len) { + return .{ true, 1 }; + } + const m = same: { + const m2 = before[b_u + 1] == after[a_u + 1]; + const m3 = before[b_u + 2] == after[a_u + 2]; + const m4 = before[b_u + 3] == after[a_u + 3]; + break :same m2 and m3 and m4; + }; + return .{ m, 4 }; + }, // follow byte or invalid high, doesn't matter, match + else => return .{ true, 1 }, + } + } + } else { + return .{ false, 0 }; + } +} + +/// Match characters backward, avoiding splitting two valid codeunits with a +/// common suffix. Once again, we are not interested in validating the text, +/// just in preventing a spurious diff which truncates Unicode. +fn equalBackward( + before: []const u8, + after: []const u8, + b_i: isize, + a_i: isize, +) struct { bool, isize } { + const b_u: usize = @intCast(b_i); + const a_u: usize = @intCast(a_i); + const b1c = before[b_u]; + const a1c = after[a_u]; + if (b1c == a1c) { + // how many codeunits might we expect? + + // different jam here! We have to match back to a lead: + switch (b1c) { + // follow byte might be a code unit sequence + 0x80...0xbf => { + // I'd rather double the offsets then deal with + // casting. Feel free to optimize... + var off: usize = 1; + var offi: isize = @intCast(off); + while (off < 4 and b_i - offi >= 0 and a_i - offi >= 0) { + const b = before[b_u - off]; + if (b != after[b_u - off]) { + // whole thing is a fail + return .{ false, 0 }; // here the offset doesn't matter + } + // check for lead byte + // since we presume well-formedness, any lead will do + if (0xc1 < b and b < 0xf5) { + return .{ true, offi + 1 }; + } + off += 1; + offi += 1; + } // since we didn't spot a plausible character, match 1 + return .{ true, 1 }; + }, // ASCII, malformed, don't care, + else => return .{ true, 1 }, + } + } else { + return .{ false, 0 }; + } +} + /// Given the location of the 'middle snake', split the diff in two parts /// and recurse. /// @param text1 Old string to be diffed. @@ -2303,8 +2417,8 @@ test "Unicode diffs" { defer deinitDiffList(allocator, &greek_diff); try testing.expectEqualDeep(@as([]const Diff, &.{ Diff.init(.equal, "αβ"), + Diff.init(.delete, "γ"), Diff.init(.insert, "δ"), - Diff.init(.equal, "γ"), }), greek_diff.items); } From 7ebdf122c04b2323dd0c07ab2576dbec806bc5ef Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 3 Jul 2024 17:50:19 -0400 Subject: [PATCH 026/176] Add some easy two-byte tests --- DiffMatchPatch.zig | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 935ece5..8701176 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2420,6 +2420,46 @@ test "Unicode diffs" { Diff.init(.delete, "γ"), Diff.init(.insert, "δ"), }), greek_diff.items); + // ө is 0xd3, 0xa9, թ is 0xd6, 0xa9 + var prefix_diff = try this.diff( + allocator, + "abө", + "abթ", + false, + ); + defer deinitDiffList(allocator, &prefix_diff); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "ab"), + Diff.init(.delete, "ө"), + Diff.init(.insert, "թ"), + }), prefix_diff.items); + var mid_diff = try this.diff( + allocator, + "αөβ", + "αթβ", + false, + ); + defer deinitDiffList(allocator, &mid_diff); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "α"), + Diff.init(.delete, "ө"), + Diff.init(.insert, "թ"), + Diff.init(.equal, "β"), + }), mid_diff.items); + + var mid_prefix = try this.diff( + allocator, + "αβλ", + "αδλ", + false, + ); + defer deinitDiffList(allocator, &mid_prefix); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "α"), + Diff.init(.delete, "β"), + Diff.init(.insert, "δ"), + Diff.init(.equal, "λ"), + }), mid_prefix.items); } test diffCleanupSemantic { From 56104d40b65738c564603cfb6326913396e09c0b Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 3 Jul 2024 18:39:45 -0400 Subject: [PATCH 027/176] Back out of overlaps if we clip a character I believe that's it for operations which perform splits on common seqs. The fun part will be getting code coverage for all these pathways. Not a clue how to handle 1,110,032 line bailouts for diffLinesToChars. --- DiffMatchPatch.zig | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 8701176..ac19864 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1535,10 +1535,10 @@ fn diffCommonOverlap(text1_in: []const u8, text2_in: []const u8) usize { // Performance analysis: https://neil.fraser.name/news/2010/11/04/ var best: usize = 0; var length: usize = 1; - while (true) { + const best_idx = idx: while (true) { const pattern = text1[text_length - length ..]; const found = std.mem.indexOf(u8, text2, pattern) orelse - return best; + break :idx best; length += found; @@ -1546,7 +1546,34 @@ fn diffCommonOverlap(text1_in: []const u8, text2_in: []const u8) usize { best = length; length += 1; } + }; + if (best_idx == 0) return best_idx; + // This would mean a truncation: lead or follow, followed by a follow + // which differs (or it would be included in our overlap) + if (text2[best_idx] >= 0x80 and is_follow(text2[best_idx + 1])) { + // back out + assert(best_idx == best); + if (!is_follow(text2[best])) { + // It's a lead, one back is fine + return best - 1; + } + best -= 1; + if (best == 0) return 0; + // It's ok to get no overlap, so we ignore malformation: + // a bunch of follows could walk back to zero, and that's + // fine with us + while (is_follow(text2[best])) { + best -= 1; + if (best == 0) return 0; + } + // should be a lead, but ASCII is fine, so + if (text2[best] < 0x80) { + return best; + } else { + return best - 1; + } } + return best_idx; } // DONE [✅]: Allocate all text in diffs to From 2f2e57ea16abf15f81db2301e83120c916fc7336 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 3 Jul 2024 22:00:36 -0400 Subject: [PATCH 028/176] Less than zero impossible for usize --- DiffMatchPatch.zig | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index ac19864..44e9279 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -346,7 +346,7 @@ fn diffHalfMatch( before: []const u8, after: []const u8, ) DiffError!?HalfMatchResult { - if (dmp.diff_timeout <= 0) { + if (dmp.diff_timeout == 0) { // Don't risk returning a non-optimal diff if we have unlimited time. return null; } @@ -624,7 +624,7 @@ fn equalForward( if (b1c == a1c) { // how many codeunits might we expect? // ASCII is easy: - if (b1c < 128) { + if (b1c < 0x80) { return .{ true, 1 }; } else { switch (b1c) { @@ -681,7 +681,6 @@ fn equalBackward( const a1c = after[a_u]; if (b1c == a1c) { // how many codeunits might we expect? - // different jam here! We have to match back to a lead: switch (b1c) { // follow byte might be a code unit sequence @@ -1435,7 +1434,7 @@ pub fn diffCleanupEfficiency( var post_ins = false; // Is there a deletion operation after the last equality. var post_del = false; - while (pointer < diffs.Count) { + while (pointer < diffs.len) { if (diffs.items[pointer].operation == .equal) { // Equality found. if (diffs.items[pointer].text.len < dmp.diff_edit_cost and (post_ins or post_del)) { // Candidate found. From 7b7cd265171385d34c551d4ecd67efd91e58e46b Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 3 Jul 2024 22:00:49 -0400 Subject: [PATCH 029/176] Add coverage step for build system --- .gitignore | 1 + build.zig | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.gitignore b/.gitignore index 68557b5..0308b4a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ zig-* .zig-* +kcov-* diff --git a/build.zig b/build.zig index dd40eb6..ee140be 100644 --- a/build.zig +++ b/build.zig @@ -32,4 +32,19 @@ pub fn build(b: *std.Build) void { const step_tests = b.addRunArtifact(tests); b.step("test", "Run diffz tests").dependOn(&step_tests.step); + + // Adds a step to generate code coverage + const cov_step = b.step("cov", "Generate coverage (kcov must be installed)"); + + const cov_run = b.addSystemCommand(&.{ + "kcov", + "--clean", + "--include-pattern=DiffMatchPatch.zig", + "--exclude-line=unreachable,expect(false)", + "kcov-output", + }); + cov_run.addArtifactArg(tests); + cov_step.dependOn(&cov_run.step); + _ = cov_run.captureStdOut(); + _ = cov_run.captureStdErr(); } From 0dbfb97f49fac433c6e8083fc77c379394ba603d Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 3 Jul 2024 22:21:18 -0400 Subject: [PATCH 030/176] Add diffIndex, diffBeforeText, diffAfterText --- DiffMatchPatch.zig | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 44e9279..835c710 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1575,6 +1575,82 @@ fn diffCommonOverlap(text1_in: []const u8, text2_in: []const u8) usize { return best_idx; } +/// loc is a location in text1, compute and return the equivalent location in +/// text2. +/// e.g. "The cat" vs "The big cat", 1->1, 5->8 +/// @param diffs List of Diff objects. +/// @param loc Location within text1. +/// @return Location within text2. +/// +pub fn diffIndex(diffs: DiffList, loc: usize) usize { + // int chars1 = 0; + // int chars2 = 0; + // int last_chars1 = 0; + // int last_chars2 = 0; + var chars1: usize = 0; + var chars2: usize = 0; + var last_chars1: usize = 0; + var last_chars2: usize = 0; + // Dummy diff + var last_diff: Diff = Diff{ .operation = .equal, .text = "" }; + for (diffs) |a_diff| { + if (a_diff.operation != .insert) { + // Equality or deletion. + chars1 += a_diff.text.len; + } + if (a_diff.operation != .delete) { + // Equality or insertion. + chars2 += a_diff.text.len; + } + if (chars1 > loc) { + // Overshot the location. + last_diff = a_diff; + break; + } + } + last_chars1 = chars1; + last_chars2 = chars2; + + if (last_diff.text.len != 0 and last_diff.operation == .delete) { + // The location was deleted. + return last_chars2; + } + // Add the remaining character length. + return last_chars2 + (loc - last_chars1); +} + +/// +/// Compute and return the source text (all equalities and deletions). +/// @param diffs List of Diff objects. +/// @return Source text. +/// +pub fn diffBeforeText(allocator: Allocator, diffs: DiffList) ![]const u8 { + var chars = ArrayListUnmanaged(u8){}; + defer chars.deinit(allocator); + for (diffs) |d| { + if (d.operation != .insert) { + try chars.appendSlice(allocator, d.text); + } + } + return chars.toOwnedSlice(allocator); +} + +/// +/// Compute and return the destination text (all equalities and insertions). +/// @param diffs List of Diff objects. +/// @return Destination text. +/// +pub fn diffAfterText(allocator: Allocator, diffs: DiffList) ![]const u8 { + var chars = ArrayListUnmanaged(u8){}; + defer chars.deinit(allocator); + for (diffs) |d| { + if (d.operation != .delete) { + try chars.appendSlice(allocator, d.text); + } + } + return chars.toOwnedSlice(allocator); +} + // DONE [✅]: Allocate all text in diffs to // not cause segfault while freeing; not a problem // at the moment because we don't free anything :( From 5a7ba7cef09d002516e072ab7a8aaf799f61bb9c Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 3 Jul 2024 22:56:35 -0400 Subject: [PATCH 031/176] Add Levenshtein distance and prettyFormat --- DiffMatchPatch.zig | 86 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 835c710..75658ac 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1619,6 +1619,63 @@ pub fn diffIndex(diffs: DiffList, loc: usize) usize { return last_chars2 + (loc - last_chars1); } +/// A struct holding bookends for `diffPrittyFormat(diffs)`. +/// +/// May include a function taking an allocator and the diff, +/// which shall return the text of the diff, appropriately munged. +pub const DiffDecorations = struct { + delete_start: []const u8 = "", + delete_end: []const u8 = "", + insert_start: []const u8 = "", + insert_end: []const u8 = "", + equals_start: []const u8 = "", + equals_end: []const u8 = "", + pre_process: ?fn (Allocator, Diff) error{OutOfMemory}![]const u8 = null, +}; + +/// Decorations for classic Xterm printing: red for delete and +/// green for insert. +pub const xterm_classic = DiffDecorations{ + .delete_start = "\x1b[91m", + .delete_end = "\x1b[m", + .insert_start = "\x1b[92m", + .insert_end = "\x1b[m", +}; + +pub fn diffPrettyFormat( + allocator: Allocator, + diffs: DiffList, + deco: DiffDecorations, +) ![]const u8 { + var out = ArrayListUnmanaged(u8){}; + defer out.deinit(allocator); + for (diffs) |d| { + const text = if (deco.pre_process) |lambda| + try lambda(allocator, d) + else + d.text; + switch (d.operation) { + .delete => { + // + try out.appendSlice(allocator, deco.delete_start); + try out.appendSlice(allocator, text); + try out.appendSlice(allocator, deco.delete_end); + }, + .insert => { + try out.appendSlice(allocator, deco.insert_start); + try out.appendSlice(allocator, text); + try out.appendSlice(allocator, deco.insert_end); + }, + .equals => { + try out.appendSlice(allocator, deco.equals_start); + try out.appendSlice(allocator, text); + try out.appendSlice(allocator, deco.equals_end); + }, + } + } + return out.toOwnedSlice(allocator); +} + /// /// Compute and return the source text (all equalities and deletions). /// @param diffs List of Diff objects. @@ -1651,6 +1708,35 @@ pub fn diffAfterText(allocator: Allocator, diffs: DiffList) ![]const u8 { return chars.toOwnedSlice(allocator); } +/// +/// Compute the Levenshtein distance; the number of inserted, deleted or +/// substituted characters. +/// @param diffs List of Diff objects. +/// @return Number of changes. +/// +pub fn diffLevenshtein(diffs: DiffList) usize { + var inserts: usize = 0; + var deletes: usize = 0; + var levenshtein: usize = 0; + for (diffs) |d| { + switch (d.operation) { + .insert => { + inserts += d.text.len; + }, + .delete => { + deletes += d.text.len; + }, + .equal => { + // A deletion and an insertion is one substitution. + levenshtein = @max(inserts, deletes); + inserts = 0; + deletes = 0; + }, + } + } + return levenshtein + @max(inserts, deletes); +} + // DONE [✅]: Allocate all text in diffs to // not cause segfault while freeing; not a problem // at the moment because we don't free anything :( From 477bd5b1ff73456a16f9f73f5768b25a99123fe7 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 3 Jul 2024 23:37:57 -0400 Subject: [PATCH 032/176] Add borrowed encodeUri routine --- DiffMatchPatch.zig | 54 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 75658ac..7b143d5 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1709,8 +1709,9 @@ pub fn diffAfterText(allocator: Allocator, diffs: DiffList) ![]const u8 { } /// -/// Compute the Levenshtein distance; the number of inserted, deleted or -/// substituted characters. +/// Compute the Levenshtein distance; the number of inserted, +/// deleted or substituted characters. +/// /// @param diffs List of Diff objects. /// @return Number of changes. /// @@ -1734,9 +1735,58 @@ pub fn diffLevenshtein(diffs: DiffList) usize { }, } } + return levenshtein + @max(inserts, deletes); } +/// Borrowed from https://github.com/elerch/aws-sdk-for-zig/blob/master/src/aws_http.zig +/// under the MIT license. Thanks! +/// +/// URI encode every byte except the unreserved characters: +/// 'A'-'Z', 'a'-'z', '0'-'9', '-', '.', '_', ' ', and '~'. +/// +/// The space character is not a reserved character in the Unidiff format: +/// https://github.com/google/diff-match-patch/wiki/Unidiff +/// +/// Each URI encoded byte is formed by a '%' and the two-digit +/// hexadecimal value of the byte. +/// +/// Letters in the hexadecimal value must be uppercase, for example "%1A". +/// +fn encodeUri(allocator: std.mem.Allocator, text: []const u8) ![]u8 { + const unreserved_marks = " -_.!~*'()"; + var encoded = try std.ArrayList(u8).initCapacity(allocator, text.len); + defer encoded.deinit(); + for (text) |c| { + const should_encode = should: { + if (std.ascii.isAlphanumeric(c)) { + break :should false; + } + for (unreserved_marks) |r| { + if (r == c) { + break :should false; + } + } + break :should true; + }; + + if (!should_encode) { + try encoded.append(c); + continue; + } + // Whatever remains, encode it + try encoded.append('%'); + const hex = try std.fmt.allocPrint( + allocator, + "{s}", + .{std.fmt.fmtSliceHexUpper(&[_]u8{c})}, + ); + defer allocator.free(hex); + try encoded.appendSlice(hex); + } + return encoded.toOwnedSlice(); +} + // DONE [✅]: Allocate all text in diffs to // not cause segfault while freeing; not a problem // at the moment because we don't free anything :( From 16a05d227ca85d51d482a6c0b3cd92b33b066bb7 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 4 Jul 2024 00:04:21 -0400 Subject: [PATCH 033/176] Code-bumming encodeUri Reduces common subsequence to a pair of comparisons, and orders test in approximate frequency of occurance (until we get to the fribbles at the end, which are just in ascending order). If I really cared I could reduce this to two u64 masks, a byte mask, a switch on the four values of that result, and a shift-and- compare. Seems excessive though. And I *like* excessive... --- DiffMatchPatch.zig | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 7b143d5..93049e2 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1742,27 +1742,43 @@ pub fn diffLevenshtein(diffs: DiffList) usize { /// Borrowed from https://github.com/elerch/aws-sdk-for-zig/blob/master/src/aws_http.zig /// under the MIT license. Thanks! /// -/// URI encode every byte except the unreserved characters: -/// 'A'-'Z', 'a'-'z', '0'-'9', '-', '.', '_', ' ', and '~'. -/// -/// The space character is not a reserved character in the Unidiff format: +/// Modified to implement Unidiff escaping, documented here: /// https://github.com/google/diff-match-patch/wiki/Unidiff /// +/// The documentation reads: +/// +/// > Special characters are encoded using %xx notation. The set of +/// > characters which are encoded matches JavaScript's `encodeURI()` +/// > function, with the exception of spaces which are not encoded. +/// +/// So we encode everything but the characters defined by Moz: +/// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURI +/// +/// These: !#$&'()*+,-./:;=?@_~ +/// +/// There is a nice contiguous run of 10 symbols between `&` and `/`, which we +/// can test in two comparisons, leaving these assorted: +/// +/// !#$:;=?@_~ +/// /// Each URI encoded byte is formed by a '%' and the two-digit /// hexadecimal value of the byte. /// /// Letters in the hexadecimal value must be uppercase, for example "%1A". /// fn encodeUri(allocator: std.mem.Allocator, text: []const u8) ![]u8 { - const unreserved_marks = " -_.!~*'()"; + const remaining_characters = "!#$:;=?@_~"; var encoded = try std.ArrayList(u8).initCapacity(allocator, text.len); defer encoded.deinit(); for (text) |c| { const should_encode = should: { - if (std.ascii.isAlphanumeric(c)) { + if (c == ' ' or std.ascii.isAlphanumeric(c)) { + break :should false; + } + if ('&' <= c and c <= '/') { break :should false; } - for (unreserved_marks) |r| { + for (remaining_characters) |r| { if (r == c) { break :should false; } From f64775999a870037727a11ee53e1cc5c9d1ad605 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 4 Jul 2024 09:58:15 -0400 Subject: [PATCH 034/176] Tests for encodeUri --- DiffMatchPatch.zig | 16 ++++++++++++++++ roadmap.md | 19 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 roadmap.md diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 93049e2..54202ba 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1803,6 +1803,22 @@ fn encodeUri(allocator: std.mem.Allocator, text: []const u8) ![]u8 { return encoded.toOwnedSlice(); } +test encodeUri { + const allocator = std.testing.allocator; + const special_chars = "!#$&'()*+,-./:;=?@_~"; + const special_encoded = try encodeUri(allocator, special_chars); + defer allocator.free(special_encoded); + try testing.expectEqualStrings(special_chars, special_encoded); + const alphaspace = " ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + const alpha_encoded = try encodeUri(allocator, alphaspace); + defer allocator.free(alpha_encoded); + try testing.expectEqualStrings(alphaspace, alpha_encoded); + const to_encode = "\"%<>[\\]^`{|}δ"; + const encodes = try encodeUri(allocator, to_encode); + defer allocator.free(encodes); + try testing.expectEqualStrings("%22%25%3C%3E%5B%5C%5D%5E%60%7B%7C%7D%CE%B4", encodes); +} + // DONE [✅]: Allocate all text in diffs to // not cause segfault while freeing; not a problem // at the moment because we don't free anything :( diff --git a/roadmap.md b/roadmap.md new file mode 100644 index 0000000..669512d --- /dev/null +++ b/roadmap.md @@ -0,0 +1,19 @@ +# Roadmap + +- [ ] Port patch +- [ ] Port match +- [ ] Diff stream + - [ ] Use Unicode characters and codepoint indices - 32 + - [ ] Implement line diff as a stream + - [ ] Also gives word diff, token diff, etc. +- [ ] Refactor: + - [ ] Diff struct becomes Edit + - [ ] DiffList stays + - [ ] New Diff struct, and DiffUnmanaged + - [ ] Namespaces subsequent operations on diffs +- [ ] Histogram? + - [ ] Imara diff has an optimized histogram: + https://github.com/pascalkuthe/imara-diff +- [ ] POSIX-diff compatible patch output? + +Covers the bases. From 3d0cb776f9bc5f63e6e03651c9a2dba5f70f245a Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 4 Jul 2024 10:02:03 -0400 Subject: [PATCH 035/176] No extra allocation for hex formatting --- DiffMatchPatch.zig | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 54202ba..30c041c 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1792,13 +1792,8 @@ fn encodeUri(allocator: std.mem.Allocator, text: []const u8) ![]u8 { } // Whatever remains, encode it try encoded.append('%'); - const hex = try std.fmt.allocPrint( - allocator, - "{s}", - .{std.fmt.fmtSliceHexUpper(&[_]u8{c})}, - ); - defer allocator.free(hex); - try encoded.appendSlice(hex); + const hexen = std.fmt.bytesToHex(&[_]u8{c}, .upper); + try encoded.appendSlice(&hexen); } return encoded.toOwnedSlice(); } From 7b650f77a4657c6bad3f6a93fbb48c1e1ee09716 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 4 Jul 2024 10:33:56 -0400 Subject: [PATCH 036/176] Factor uriEncode to use writeUriEncoded This will avoid making intermediate copies. --- DiffMatchPatch.zig | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 30c041c..dcad2d9 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1766,10 +1766,9 @@ pub fn diffLevenshtein(diffs: DiffList) usize { /// /// Letters in the hexadecimal value must be uppercase, for example "%1A". /// -fn encodeUri(allocator: std.mem.Allocator, text: []const u8) ![]u8 { +fn writeUriEncoded(writer: anytype, text: []const u8) !usize { const remaining_characters = "!#$:;=?@_~"; - var encoded = try std.ArrayList(u8).initCapacity(allocator, text.len); - defer encoded.deinit(); + var written: usize = 0; for (text) |c| { const should_encode = should: { if (c == ' ' or std.ascii.isAlphanumeric(c)) { @@ -1787,15 +1786,25 @@ fn encodeUri(allocator: std.mem.Allocator, text: []const u8) ![]u8 { }; if (!should_encode) { - try encoded.append(c); + try writer.writeByte(c); + written += 1; continue; } // Whatever remains, encode it - try encoded.append('%'); + try writer.writeByte('%'); + written += 1; const hexen = std.fmt.bytesToHex(&[_]u8{c}, .upper); - try encoded.appendSlice(&hexen); + written += try writer.write(&hexen); } - return encoded.toOwnedSlice(); + return written; +} + +fn encodeUri(allocator: std.mem.Allocator, text: []const u8) ![]u8 { + var charlist = try std.ArrayList(u8).initCapacity(allocator, text.len); + defer charlist.deinit(); + const writer = charlist.writer(); + _ = try writeUriEncoded(writer, text); + return charlist.toOwnedSlice(); } test encodeUri { From b36c8bf4ac0377aa255e84629974a72446c332fe Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 4 Jul 2024 10:47:02 -0400 Subject: [PATCH 037/176] Break pretty format into writer and string-er Same deal: if the user has a writer available, there is no need to make an intermediate string. --- DiffMatchPatch.zig | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index dcad2d9..56af1d3 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1642,6 +1642,8 @@ pub const xterm_classic = DiffDecorations{ .insert_end = "\x1b[m", }; +/// Return text representing a pretty-formatted `DiffList`. +/// See `DiffDecorations` for how to customize this output. pub fn diffPrettyFormat( allocator: Allocator, diffs: DiffList, @@ -1649,31 +1651,50 @@ pub fn diffPrettyFormat( ) ![]const u8 { var out = ArrayListUnmanaged(u8){}; defer out.deinit(allocator); + const writer = out.writer(); + _ = try writeDiffPrettyFormat(allocator, writer, diffs, deco); + return out.toOwnedSlice(allocator); +} + +/// Write a pretty-formatted `DiffList` to `writer`. The allocator +/// is only used if a custom text formatter is defined for +/// `DiffDecorations`. Returns number of bytes written. +pub fn writeDiffPrettyFormat( + allocator: Allocator, + writer: anytype, + diffs: DiffList, + deco: DiffDecorations, +) !usize { + var written: usize = 0; for (diffs) |d| { const text = if (deco.pre_process) |lambda| try lambda(allocator, d) else d.text; + defer { + if (deco.pre_process) |_| + allocator.free(text); + } switch (d.operation) { .delete => { // - try out.appendSlice(allocator, deco.delete_start); - try out.appendSlice(allocator, text); - try out.appendSlice(allocator, deco.delete_end); + written += try writer.write(deco.delete_start); + written += try writer.write(text); + written += try writer.write(deco.delete_end); }, .insert => { - try out.appendSlice(allocator, deco.insert_start); - try out.appendSlice(allocator, text); - try out.appendSlice(allocator, deco.insert_end); + written += try writer.write(deco.insert_start); + written += try writer.write(text); + written += try writer.write(deco.insert_end); }, .equals => { - try out.appendSlice(allocator, deco.equals_start); - try out.appendSlice(allocator, text); - try out.appendSlice(allocator, deco.equals_end); + written += try writer.write(deco.equals_start); + written += try writer.write(text); + written += try writer.write(deco.equals_end); }, } } - return out.toOwnedSlice(allocator); + return written; } /// From 22c4c8466370b6d9769b8ceb3afd1a8fef719ae2 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 4 Jul 2024 10:56:49 -0400 Subject: [PATCH 038/176] Add Patch struct --- DiffMatchPatch.zig | 25 +++++++++++++++++++++++++ roadmap.md | 1 + 2 files changed, 26 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 56af1d3..07199ad 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -18,6 +18,8 @@ pub fn deinitDiffList(allocator: Allocator, diffs: *DiffList) void { } } +/// Free a range of Diffs inside a list. Used during cleanups and +/// edits. fn freeRangeDiffList( allocator: Allocator, diffs: *DiffList, @@ -34,6 +36,8 @@ fn freeRangeDiffList( /// DMP with default configuration options pub const default = DiffMatchPatch{}; +/// Represents a single edit operation. +/// TODO rename this Edit pub const Diff = struct { pub const Operation = enum { insert, @@ -63,6 +67,27 @@ pub const Diff = struct { } }; +pub const Patch = struct { + /// Diffs to be applied + diffs: DiffList, // TODO This should be a Diff + /// Start of patch in before text + start1: usize, + length1: usize, + /// Start of patch in after text + start2: usize, + length2: usize, + + pub fn toString(self: Patch) ![]const u8 { + // TODO + _ = self; + } + + pub fn writeTo(writer: anytype) !usize { + // TODO + _ = writer; + } +}; + /// Number of milliseconds to map a diff before giving up (0 for infinity). diff_timeout: u64 = 1000, /// Cost of an empty edit operation in terms of edit characters. diff --git a/roadmap.md b/roadmap.md index 669512d..18b50e3 100644 --- a/roadmap.md +++ b/roadmap.md @@ -15,5 +15,6 @@ - [ ] Imara diff has an optimized histogram: https://github.com/pascalkuthe/imara-diff - [ ] POSIX-diff compatible patch output? +- [ ] Delta functions? They aren't used internally. Covers the bases. From ef5dda5fed01caf18b6317e274e60a3dfdc8857f Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 4 Jul 2024 11:45:38 -0400 Subject: [PATCH 039/176] Port main makePatch function Untested as yet. --- DiffMatchPatch.zig | 113 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 105 insertions(+), 8 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 07199ad..301a2de 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -6,6 +6,7 @@ const assert = std.debug.assert; const Allocator = std.mem.Allocator; const ArrayListUnmanaged = std.ArrayListUnmanaged; const DiffList = ArrayListUnmanaged(Diff); +const PatchList = ArrayListUnmanaged(Patch); /// Deinit an `ArrayListUnmanaged(Diff)` and the allocated slices of /// text in each `Diff`. @@ -65,17 +66,24 @@ pub const Diff = struct { pub fn eql(a: Diff, b: Diff) bool { return a.operation == b.operation and std.mem.eql(u8, a.text, b.text); } + + pub fn clone(self: Diff, allocator: Allocator) !Diff { + return Diff{ + .operation = self.operation, + .text = try allocator.dupe(u8, self.text), + }; + } }; pub const Patch = struct { /// Diffs to be applied diffs: DiffList, // TODO This should be a Diff /// Start of patch in before text - start1: usize, - length1: usize, + start1: usize = 0, + length1: usize = 0, /// Start of patch in after text - start2: usize, - length2: usize, + start2: usize = 0, + length2: usize = 0, pub fn toString(self: Patch) ![]const u8 { // TODO @@ -1438,7 +1446,7 @@ fn diffCleanupSemanticScore(one: []const u8, two: []const u8) usize { } /// Reduce the number of edits by eliminating operationally trivial -/// equalities. +/// equalities. TODO this needs tests pub fn diffCleanupEfficiency( dmp: DiffMatchPatch, allocator: std.mem.Allocator, @@ -1648,6 +1656,9 @@ pub fn diffIndex(diffs: DiffList, loc: usize) usize { /// /// May include a function taking an allocator and the diff, /// which shall return the text of the diff, appropriately munged. +/// Note that if the function is provided, all text returned will +/// be freed, so it should always return a copy whether or not +/// edits are needed. pub const DiffDecorations = struct { delete_start: []const u8 = "", delete_end: []const u8 = "", @@ -1681,7 +1692,7 @@ pub fn diffPrettyFormat( return out.toOwnedSlice(allocator); } -/// Write a pretty-formatted `DiffList` to `writer`. The allocator +/// Write a pretty-formatted `DiffList` to `writer`. The `Allocator` /// is only used if a custom text formatter is defined for /// `DiffDecorations`. Returns number of bytes written. pub fn writeDiffPrettyFormat( @@ -1724,7 +1735,7 @@ pub fn writeDiffPrettyFormat( /// /// Compute and return the source text (all equalities and deletions). -/// @param diffs List of Diff objects. +/// @param diffs List of `Diff` objects. /// @return Source text. /// pub fn diffBeforeText(allocator: Allocator, diffs: DiffList) ![]const u8 { @@ -1740,7 +1751,7 @@ pub fn diffBeforeText(allocator: Allocator, diffs: DiffList) ![]const u8 { /// /// Compute and return the destination text (all equalities and insertions). -/// @param diffs List of Diff objects. +/// @param diffs List of `Diff` objects. /// @return Destination text. /// pub fn diffAfterText(allocator: Allocator, diffs: DiffList) ![]const u8 { @@ -1785,6 +1796,92 @@ pub fn diffLevenshtein(diffs: DiffList) usize { return levenshtein + @max(inserts, deletes); } +pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !PatchList { + // TODO maybe add a .own and .borrow enum, sometimes the diffs will be + // created internally and we can just move them? That would be an internal + // function, public `makePatch` would use .own + const patches = PatchList{}; + if (diffs.items.len == 0) { + return patches; // Empty diff means empty patchlist + } + + var patch = Patch{}; + var char_count1 = 0; + var char_count2 = 0; + + // This avoids freeing the original copy of the text: + var first_patch = true; + var prepatch_text = text; + defer { + if (!first_patch) + allocator.free(prepatch_text); + } + var postpatch = try std.ArrayList(u8).initCapacity(allocator, text.len); + defer postpatch.deinit(); + try postpatch.appendSlice(text); + for (diffs) |a_diff| { + if (patch.diffs.items.len == 0 and a_diff.operation != .equal) { + patch.start1 = char_count1; + patch.start2 = char_count2; + } + switch (a_diff.operation) { + .insert => { + try patch.diffs.append(allocator, a_diff.clone(allocator)); + patch.length2 += a_diff.text.len; + try postpatch.insertSlice(char_count2, a_diff.text); + }, + .delete => { + // + try patch.diffs.append(allocator, a_diff.clone(allocator)); + patch.length1 += a_diff.text.len; + try postpatch.replaceRange(char_count2, a_diff.text.len, .{}); + }, + .equal => { + // + if (a_diff.text.len <= 2 * @This().patch_margin and patch.diffs.items.len != 0 and a_diff != diffs.items[diffs.items.len]) { + // Small equality inside a patch. + try patch.diffs.append(allocator, try a_diff.clone(allocator)); + patch.length1 += a_diff.text.len; + patch.length2 += a_diff.text.len; + } + if (a_diff.text.len >= 2 * @This().patch_margin) { + // Time for a new patch. + if (patch.diffs.items.len != 0) { + // patchAddContext(patch, prepatch_text); + try patches.append(allocator, patch); + patch = Patch{}; + // Unlike Unidiff, our patch lists have a rolling context. + // https://github.com/google/diff-match-patch/wiki/Unidiff + // Update prepatch text & pos to reflect the application of the + // just completed patch. + if (first_patch) { + // no free on first + first_patch = false; + } else { + allocator.free(prepatch_text); + } + prepatch_text = try allocator.dupe(u8, postpatch.items); + char_count1 = char_count2; + } + } + }, + } + // Update the current character count. + if (a_diff.operation != .insert) { + char_count1 += a_diff.text.len; + } + if (a_diff.operation != .remove) { + char_count2 += a_diff.text.len; + } + } // end for loop + + // Pick up the leftover patch if not empty. + if (patch.diffs.items.len != 0) { + // patchAddContext(patch, prepatch_text); + try patches.append(allocator, patch); + } +} + /// Borrowed from https://github.com/elerch/aws-sdk-for-zig/blob/master/src/aws_http.zig /// under the MIT license. Thanks! /// From eeaf42fe32d5dca5541dbd7a2548e34c658ff395 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 4 Jul 2024 12:17:58 -0400 Subject: [PATCH 040/176] Adds patchAddContext Which finishes the port of the patch creation algorithm. I've accumulated some test debt here, probably worth paying that down before finishing up the match code and starting to refactor this thing into shape. --- DiffMatchPatch.zig | 75 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 301a2de..82e2fa9 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1796,6 +1796,77 @@ pub fn diffLevenshtein(diffs: DiffList) usize { return levenshtein + @max(inserts, deletes); } +//| PATCH FUNCTIONS + +/// +/// Increase the context until it is unique, but don't let the pattern +/// expand beyond DiffMatchPatch.match_max_bits. +/// +/// @param patch The patch to grow. +/// @param text Source text. +fn patchAddContext(allocator: Allocator, patch: *Patch, text: []const u8) !void { + // + if (text.len == 0) return; + var pattern = text[patch.start2 .. patch.start2 + patch.length1]; + var padding = 0; + if (false) { // XXX + pattern = ""; + padding = 0; + } + while (std.mem.indexOf(u8, text, pattern) != std.mem.lastIndexOf(u8, text, pattern) and pattern.len < @This().match_max_bits - (2 * @This().patch_margin)) { + // + padding += @This().patch_margin; + const pat_start = @max(0, patch.start2 - padding); + const pat_end = pat_start + @min(text.len, patch.start2 + patch.length1 + padding); + pattern = text[pat_start..pat_end]; + } + // Add one chunk for good luck. + padding += @This().patch_margin; + // Add the prefix. + const prefix = pre: { + const pre_start = @max(0, patch.start2 - padding); + const pre_end = pre_start + patch.start2; + break :pre text[pre_start..pre_end]; + }; + if (prefix.len != 0) { + try patch.diffs.append( + allocator, + Diff{ + .operation = .equal, + .text = try allocator.dupe(u8, prefix), + }, + ); + } + // Add the suffix. + const suffix = post: { + const post_start = patch.start2 + patch.length1; + const post_end = post_start + @min(text.len, patch.start2 + patch.length1 + padding); + break :post text[post_start..post_end]; + }; + if (suffix.len != 0) { + try patch.diffs.append( + allocator, + Diff{ + .operation = .equal, + .text = try allocator.dupe(u8, suffix), + }, + ); + } + // Roll back the start points. + patch.start1 -= prefix.len; + patch.start2 -= prefix.len; + // Extend the lengths. + patch.length1 += prefix.len + suffix.len; + patch.length2 += prefix.len + suffix.len; +} + +/// +/// Compute a list of patches to turn text1 into text2. +/// text2 is not provided, diffs are the delta between text1 and text2. +/// +/// @param text1 Old text. +/// @param diffs Array of Diff objects for text1 to text2. +/// @return List of Patch objects. pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !PatchList { // TODO maybe add a .own and .borrow enum, sometimes the diffs will be // created internally and we can just move them? That would be an internal @@ -1847,7 +1918,7 @@ pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !Patch if (a_diff.text.len >= 2 * @This().patch_margin) { // Time for a new patch. if (patch.diffs.items.len != 0) { - // patchAddContext(patch, prepatch_text); + try patchAddContext(allocator, patch, prepatch_text); try patches.append(allocator, patch); patch = Patch{}; // Unlike Unidiff, our patch lists have a rolling context. @@ -1877,7 +1948,7 @@ pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !Patch // Pick up the leftover patch if not empty. if (patch.diffs.items.len != 0) { - // patchAddContext(patch, prepatch_text); + try patchAddContext(allocator, patch, prepatch_text); try patches.append(allocator, patch); } } From 5fb7299378b952f4b15fc05bd4399981e424a3df Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 4 Jul 2024 13:42:05 -0400 Subject: [PATCH 041/176] Add ownership handling to makePatch Some of the functions which produce a patch, produce the Diffs as well. Those functions can call makePatchInternal with the .own enum, and then we don't have to copy the Diffs: the unused ones are freed, and the new ones are allocated, while the edit diffs get moved to the Patch. Then the original DiffList can be deinitialized without needing to also free the Diff texts. --- DiffMatchPatch.zig | 46 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 82e2fa9..71d9667 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1860,14 +1860,28 @@ fn patchAddContext(allocator: Allocator, patch: *Patch, text: []const u8) !void patch.length2 += prefix.len + suffix.len; } +/// Determines how to handle Diffs in a patch. Functions which create +/// the diffs internally can use `.own`: the Diffs will be copied to +/// the patch list, new ones allocated, and old ones freed. Then call +/// `deinit` on the DiffList, but not `deinitDiffList`. This *must not* +/// be used if the DiffList is not immediately freed, because some of +/// the diffs will contain spuriously empty text. /// -/// Compute a list of patches to turn text1 into text2. -/// text2 is not provided, diffs are the delta between text1 and text2. -/// -/// @param text1 Old text. -/// @param diffs Array of Diff objects for text1 to text2. +/// Functions which operate on an existing DiffList should use `.copy`: +/// as the name indicates, copies of the Diffs will be made, and the +/// original memory must be freed separately. +const DiffHandling = enum { + copy, + own, +}; + /// @return List of Patch objects. -pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !PatchList { +fn makePatchInternal( + allocator: Allocator, + text: []const u8, + diffs: DiffList, + diff_act: DiffHandling, +) !PatchList { // TODO maybe add a .own and .borrow enum, sometimes the diffs will be // created internally and we can just move them? That would be an internal // function, public `makePatch` would use .own @@ -1897,13 +1911,15 @@ pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !Patch } switch (a_diff.operation) { .insert => { - try patch.diffs.append(allocator, a_diff.clone(allocator)); + const d = if (diff_act == .copy) a_diff.clone(allocator) else a_diff; + try patch.diffs.append(allocator, d); patch.length2 += a_diff.text.len; try postpatch.insertSlice(char_count2, a_diff.text); }, .delete => { // - try patch.diffs.append(allocator, a_diff.clone(allocator)); + const d = if (diff_act == .copy) a_diff.clone(allocator) else a_diff; + try patch.diffs.append(allocator, d); patch.length1 += a_diff.text.len; try postpatch.replaceRange(char_count2, a_diff.text.len, .{}); }, @@ -1918,6 +1934,10 @@ pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !Patch if (a_diff.text.len >= 2 * @This().patch_margin) { // Time for a new patch. if (patch.diffs.items.len != 0) { + // free the Diff if we own it + if (diff_act == .own) { + allocator.free(a_diff.text); + } try patchAddContext(allocator, patch, prepatch_text); try patches.append(allocator, patch); patch = Patch{}; @@ -1953,6 +1973,16 @@ pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !Patch } } +/// +/// Compute a list of patches to turn text1 into text2. +/// text2 is not provided, diffs are the delta between text1 and text2. +/// +/// @param text1 Old text. +/// @param diffs Array of Diff objects for text1 to text2. +pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !PatchList { + try makePatchInternal(allocator, text, diffs, .copy); +} + /// Borrowed from https://github.com/elerch/aws-sdk-for-zig/blob/master/src/aws_http.zig /// under the MIT license. Thanks! /// From 6d477adfd8fc8d98c070c342a7f9ae59edaf23e6 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 4 Jul 2024 14:06:49 -0400 Subject: [PATCH 042/176] Add clone for patches and patch lists Now I just need to circle back and write the match algorithm, then use it in patchApply. A quick tidy up, a refactor, and test coverage, then we're good to go. --- DiffMatchPatch.zig | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 71d9667..9f0a472 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -85,15 +85,35 @@ pub const Patch = struct { start2: usize = 0, length2: usize = 0, - pub fn toString(self: Patch) ![]const u8 { + pub fn toString(patch: Patch) ![]const u8 { // TODO - _ = self; + _ = patch; } pub fn writeTo(writer: anytype) !usize { // TODO _ = writer; } + + /// Make a clone of the Patch, including all Diffs. + pub fn clone(patch: Patch, allocator: Allocator) !Patch { + var new_diffs = DiffList{}; + new_diffs.initCapacity(allocator, patch.diffs.items.len); + for (patch.diffs) |a_diff| { + try new_diffs.append(try a_diff.clone(allocator)); + } + return Patch{ + .diffs = new_diffs, + .start1 = patch.start1, + .length1 = patch.length1, + .start2 = patch.start2, + .length2 = patch.length2, + }; + } + + pub fn deinit(patch: *Patch, allocator: Allocator) void { + deinitDiffList(allocator, patch.diffs); + } }; /// Number of milliseconds to map a diff before giving up (0 for infinity). @@ -1983,6 +2003,22 @@ pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !Patch try makePatchInternal(allocator, text, diffs, .copy); } +// TODO other makePatch methods... + +/// +/// Given an array of patches, return another array that is identical. +/// @param patches Array of Patch objects. +/// @return Array of Patch objects. +fn patchListClone(allocator: Allocator, patches: PatchList) !PatchList { + var new_patches = PatchList{}; + new_patches.initCapacity(allocator, patches.items.len); + for (patches) |patch| { + try new_patches.append(allocator, try patch.clone(allocator)); + } + return new_patches; +} + +/// /// Borrowed from https://github.com/elerch/aws-sdk-for-zig/blob/master/src/aws_http.zig /// under the MIT license. Thanks! /// From 90604779f5d975a2e84dc1b9b9a19e6411335a13 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 4 Jul 2024 17:38:55 -0400 Subject: [PATCH 043/176] Add match section It's number-crunchy code, but it doesn't do a lot of allocation. There may be typos, it would surprise me if there are none in fact, but other than that, it should work fine. That leaves adding patchApply, translating enough unit tests to achieve full coverage, and then refactoring the surface area to be a more ergo- nomic. --- DiffMatchPatch.zig | 187 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 182 insertions(+), 5 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 9f0a472..465f498 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -122,21 +122,26 @@ diff_timeout: u64 = 1000, diff_edit_cost: u16 = 4, /// At what point is no match declared (0.0 = perfection, 1.0 = very loose). -match_threshold: f32 = 0.5, +/// This defaults to 0.05, on the premise that the library will mostly be +/// used in cases where failure is better than a bad patch application. +match_threshold: f32 = 0.05, + /// How far to search for a match (0 = exact location, 1000+ = broad match). /// A match this many characters away from the expected location will add /// 1.0 to the score (0.0 is a perfect match). match_distance: u32 = 1000, -/// The number of bits in an int. -match_max_bits: u16 = 32, + +/// The number of bits in a usize. +match_max_bits: u8 = 64, /// When deleting a large block of text (over ~64 characters), how close /// do the contents have to be to match the expected contents. (0.0 = /// perfection, 1.0 = very loose). Note that Match_Threshold controls /// how closely the end points of a delete need to match. patch_delete_threshold: f32 = 0.5, + /// Chunk size for context length. -patch_margin: u16 = 4, +patch_margin: u8 = 4, pub const DiffError = error{OutOfMemory}; @@ -1816,6 +1821,178 @@ pub fn diffLevenshtein(diffs: DiffList) usize { return levenshtein + @max(inserts, deletes); } +//| MATCH FUNCTIONS + +/// Locate the best instance of 'pattern' in 'text' near 'loc'. +/// Returns -1 if no match found. +/// @param text The text to search. +/// @param pattern The pattern to search for. +/// @param loc The location to search around. +/// @return Best match index or -1. +pub fn matchMain(allocator: Allocator, text: []const u8, pattern: []const u8, passed_loc: usize) ?usize { + // Clamp the loc to fit within text. + const loc = @min(passed_loc, text.len); + if (std.mem.eql(u8, text, pattern)) { + // Shortcut (potentially not guaranteed by the algorithm) + // TODO would be good to know what the above means... + return 0; + } else if (text.len == 0) { + // Nothing to match. + return null; + } else if (loc + pattern.len <= text.len and std.mem.eql(u8, text[loc..pattern.length], pattern)) { + // Perfect match at the perfect spot! (Includes case of null pattern) + return loc; + } else { + // Do a fuzzy compare. + // return match_bitap(allocator, text, pattern, loc); + } + _ = allocator; +} + +/// Locate the best instance of 'pattern' in 'text' near 'loc' using the +/// Bitap algorithm. Returns -1 if no match found. +/// @param text The text to search. +/// @param pattern The pattern to search for. +/// @param loc The location to search around. +/// @return Best match index or -1. +fn matchBitap( + allocator: Allocator, + text: []const u8, + pattern: []const u8, + loc: usize, +) ?usize { + // TODO decide what to do here: + // assert (Match_MaxBits == 0 || pattern.Length <= Match_MaxBits) + // : "Pattern too long for this application."; + // Initialise the alphabet. + var map = try matchAlphabet(allocator, pattern); + defer map.deinit(); + // Highest score beyond which we give up. + var threshold = @This().threshold; + // Is there a nearby exact match? (speedup) + var best_loc = std.mem.indexOfPos(u8, text, pattern); + if (best_loc) |best| { + threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + } + // What about in the other direction? (speedup) + const trunc_text = text[0..@min(loc + pattern.len, text.len)]; + best_loc = std.mem.lastIndexOf(u8, trunc_text, pattern); + if (best_loc) |best| { + threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + } + // Initialise the bit arrays. + const shift: u6 = @intCast(pattern.len - 1); + const matchmask = 1 << shift; + best_loc = null; + var bin_min: usize = undefined; + var bin_mid: usize = undefined; + var bin_max = pattern.len + text.len; + // null last_rd to simplying freeing memory + var last_rd: []usize = try allocator.alloct(usize, 0); + for (0..pattern.len) |d| { + // Scan for the best match; each iteration allows for one more error. + // Run a binary search to determine how far from 'loc' we can stray at + // this error level. + bin_min = 0; + bin_mid = bin_max; + while (bin_min < bin_mid) { + if (matchBitapScore(d, loc + bin_mid, loc, pattern) <= threshold) { + bin_min = bin_mid; + } else { + bin_max = bin_mid; + } + bin_mid = (bin_max - bin_min) / 2 + bin_min; + } + // Use the result from this iteration as the maximum for the next. + bin_max = bin_mid; + var start = @max(1, loc - bin_mid + 1); + const finish = @min(loc + bin_mid, text.len) + pattern.len; + var rd: []usize = allocator.alloc(usize, finish + 2); + const dshift: u6 = @intCast(d); + rd[finish + 1] = (1 << dshift) - 1; + var j = finish; + while (j >= start) : (j -= 1) { + const char_match: usize = if (text.len <= j - 1 or !map.contains(text[j - 1])) + // Out of range. + 0 + else + map.get(text[j - 1]); + if (d == 0) { + // First pass: exact match. + rd[j] = ((rd[j + 1] << 1) | 1) & char_match; + } else { + // Subsequent passes: fuzzy match. + rd[j] = ((rd[j + 1] << 1) | 1) & char_match | (((last_rd[j + 1] | last_rd[j]) << 1) | 1) | last_rd[j + 1]; + } + if ((rd[j] & matchmask) != 0) { + const score = matchBitapScore(d, j - 1, loc, pattern); + // This match will almost certainly be better than any existing + // match. But check anyway. + if (score <= threshold) { + // Told you so. + threshold = score; + best_loc = j - 1; + if (best_loc > loc) { + // When passing loc, don't exceed our current distance from loc. + start = @max(1, 2 * loc - best_loc); + } else { + // Already passed loc, downhill from here on in. + break; + } + } + } + } + if (matchBitapScore(d + 1, loc, loc, pattern) > threshold) { + // No hope for a (better) match at greater error levels. + break; + } + allocator.free(last_rd); + last_rd = rd; + } + allocator.free(last_rd); + return best_loc; +} + +/// Compute and return the score for a match with e errors and x location. +/// @param e Number of errors in match. +/// @param x Location of match. +/// @param loc Expected location of match. +/// @param pattern Pattern being sought. +/// @return Overall score for match (0.0 = good, 1.0 = bad). +fn matchBitapScore(e: usize, x: usize, loc: usize, pattern: []const u8) f64 { + const e_float: f32 = @floatFromInt(e); + const len_float: f32 = @floatFromInt(pattern.len); + const accuracy = e_float / len_float; + const proximity = if (loc >= x) loc - x else x - loc; + if (@This().match_distance == 0) { + // Dodge divide by zero + if (proximity == 0) + return accuracy + else + return 1.0; + } + const float_match: f64 = @floatFromInt(@This().match_distance); + return accuracy + (proximity / float_match); +} + +/// Initialise the alphabet for the Bitap algorithm. +/// @param pattern The text to encode. +/// @return Hash of character locations. +fn matchAlphabet(allocator: Allocator, pattern: []const u8) !std.HashMap(u8, usize) { + var map = std.HashMap(u8, usize).init(allocator); + for (pattern) |c| { + if (!map.contains(c)) { + try map.put(c, 0); + } + } + for (pattern, 0..) |c, i| { + const shift: u6 = @intCast(pattern.len - i - 1); + const value: usize = map.get(c) | (1 << shift); + try map.put(c, value); + } + return map; +} + //| PATCH FUNCTIONS /// @@ -2034,7 +2211,7 @@ fn patchListClone(allocator: Allocator, patches: PatchList) !PatchList { /// So we encode everything but the characters defined by Moz: /// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURI /// -/// These: !#$&'()*+,-./:;=?@_~ +/// These: !#$&'()*+,-./:;=?@_~ (and alphanumeric ASCII) /// /// There is a nice contiguous run of 10 symbols between `&` and `/`, which we /// can test in two comparisons, leaving these assorted: From 49339a9f484fc5cbafbeb5597123fbf4da611f8c Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 4 Jul 2024 18:29:22 -0400 Subject: [PATCH 044/176] Put fields at top of file/struct --- DiffMatchPatch.zig | 64 ++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 465f498..46ff997 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -8,6 +8,37 @@ const ArrayListUnmanaged = std.ArrayListUnmanaged; const DiffList = ArrayListUnmanaged(Diff); const PatchList = ArrayListUnmanaged(Patch); +//| Fields + +/// Number of milliseconds to map a diff before giving up (0 for infinity). +diff_timeout: u64 = 1000, +/// Cost of an empty edit operation in terms of edit characters. +diff_edit_cost: u16 = 4, + +/// At what point is no match declared (0.0 = perfection, 1.0 = very loose). +/// This defaults to 0.05, on the premise that the library will mostly be +/// used in cases where failure is better than a bad patch application. +match_threshold: f32 = 0.05, + +/// How far to search for a match (0 = exact location, 1000+ = broad match). +/// A match this many characters away from the expected location will add +/// 1.0 to the score (0.0 is a perfect match). +match_distance: u32 = 1000, + +/// The number of bits in a usize. +match_max_bits: u8 = 64, + +/// When deleting a large block of text (over ~64 characters), how close +/// do the contents have to be to match the expected contents. (0.0 = +/// perfection, 1.0 = very loose). Note that Match_Threshold controls +/// how closely the end points of a delete need to match. +patch_delete_threshold: f32 = 0.5, + +/// Chunk size for context length. +patch_margin: u8 = 4, + +//| Allocation Management Helpers + /// Deinit an `ArrayListUnmanaged(Diff)` and the allocated slices of /// text in each `Diff`. pub fn deinitDiffList(allocator: Allocator, diffs: *DiffList) void { @@ -116,33 +147,6 @@ pub const Patch = struct { } }; -/// Number of milliseconds to map a diff before giving up (0 for infinity). -diff_timeout: u64 = 1000, -/// Cost of an empty edit operation in terms of edit characters. -diff_edit_cost: u16 = 4, - -/// At what point is no match declared (0.0 = perfection, 1.0 = very loose). -/// This defaults to 0.05, on the premise that the library will mostly be -/// used in cases where failure is better than a bad patch application. -match_threshold: f32 = 0.05, - -/// How far to search for a match (0 = exact location, 1000+ = broad match). -/// A match this many characters away from the expected location will add -/// 1.0 to the score (0.0 is a perfect match). -match_distance: u32 = 1000, - -/// The number of bits in a usize. -match_max_bits: u8 = 64, - -/// When deleting a large block of text (over ~64 characters), how close -/// do the contents have to be to match the expected contents. (0.0 = -/// perfection, 1.0 = very loose). Note that Match_Threshold controls -/// how closely the end points of a delete need to match. -patch_delete_threshold: f32 = 0.5, - -/// Chunk size for context length. -patch_margin: u8 = 4, - pub const DiffError = error{OutOfMemory}; /// Find the differences between two texts. @@ -2079,9 +2083,6 @@ fn makePatchInternal( diffs: DiffList, diff_act: DiffHandling, ) !PatchList { - // TODO maybe add a .own and .borrow enum, sometimes the diffs will be - // created internally and we can just move them? That would be an internal - // function, public `makePatch` would use .own const patches = PatchList{}; if (diffs.items.len == 0) { return patches; // Empty diff means empty patchlist @@ -2124,7 +2125,8 @@ fn makePatchInternal( // if (a_diff.text.len <= 2 * @This().patch_margin and patch.diffs.items.len != 0 and a_diff != diffs.items[diffs.items.len]) { // Small equality inside a patch. - try patch.diffs.append(allocator, try a_diff.clone(allocator)); + const d = if (diff_act == .copy) a_diff.clone(allocator) else a_diff; + try patch.diffs.append(allocator, d); patch.length1 += a_diff.text.len; patch.length2 += a_diff.text.len; } From 1dea68a642de1e0d8ec90ac6cef2db4514bf5950 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 4 Jul 2024 19:04:04 -0400 Subject: [PATCH 045/176] Don't clip codepoints in patch context UTF-8 is an unusually demanding encoding, not gonna lie. --- DiffMatchPatch.zig | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 46ff997..6558f3a 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2006,26 +2006,29 @@ fn matchAlphabet(allocator: Allocator, pattern: []const u8) !std.HashMap(u8, usi /// @param patch The patch to grow. /// @param text Source text. fn patchAddContext(allocator: Allocator, patch: *Patch, text: []const u8) !void { - // if (text.len == 0) return; - var pattern = text[patch.start2 .. patch.start2 + patch.length1]; + // TODO the fixup logic here might make patterns too large? var padding = 0; - if (false) { // XXX - pattern = ""; - padding = 0; - } - while (std.mem.indexOf(u8, text, pattern) != std.mem.lastIndexOf(u8, text, pattern) and pattern.len < @This().match_max_bits - (2 * @This().patch_margin)) { - // - padding += @This().patch_margin; - const pat_start = @max(0, patch.start2 - padding); - const pat_end = pat_start + @min(text.len, patch.start2 + patch.length1 + padding); - pattern = text[pat_start..pat_end]; + { // Grow the pattern around the patch until unique, to set padding amount. + var pattern = text[patch.start2 .. patch.start2 + patch.length1]; + const max_width: usize = @This().match_max_bits - (2 * @This().patch_margin); + while (std.mem.indexOf(u8, text, pattern) != std.mem.lastIndexOf(u8, text, pattern) and pattern.len < max_width) { + padding += @This().patch_margin; + const pat_start = @max(0, patch.start2 - padding); + const pat_end = pat_start + @min(text.len, patch.start2 + patch.length1 + padding); + pattern = text[pat_start..pat_end]; + } } // Add one chunk for good luck. padding += @This().patch_margin; // Add the prefix. const prefix = pre: { - const pre_start = @max(0, patch.start2 - padding); + var pre_start = @max(0, patch.start2 - padding); + // Make sure we're not breaking a codepoint. + while (is_follow(text[pre_start]) and pre_start > 0) { + pre_start -= 1; + } // Assuming we did everything else right, pre_end should be + // properly placed. const pre_end = pre_start + patch.start2; break :pre text[pre_start..pre_end]; }; @@ -2041,7 +2044,18 @@ fn patchAddContext(allocator: Allocator, patch: *Patch, text: []const u8) !void // Add the suffix. const suffix = post: { const post_start = patch.start2 + patch.length1; - const post_end = post_start + @min(text.len, patch.start2 + patch.length1 + padding); + // In case we messed up somewhere: + assert(!is_follow(text[post_start])); + var post_end = post_start + @min(text.len, patch.start2 + patch.length1 + padding); + // Prevent broken codepoints here as well: Lead bytes, or follow with another follow + while (!std.ascii.isASCII(text[post_end]) and post_end + 1 < text.len and is_follow(text[post_end + 1])) { + post_end += 1; + // Special case: penultimate with another follow at end + if (post_end + 2 == text.len and is_follow(text[post_end + 1])) { + post_end += 1; + break; // Not actually necessary, but polite. + } + } break :post text[post_start..post_end]; }; if (suffix.len != 0) { From a7cc1b2acc610f661494b42b18f8c474f9f9bba0 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 4 Jul 2024 20:20:02 -0400 Subject: [PATCH 046/176] Mid-function putter out Just out of steam on all this. More later. --- DiffMatchPatch.zig | 214 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 212 insertions(+), 2 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 6558f3a..0c9be66 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -8,6 +8,10 @@ const ArrayListUnmanaged = std.ArrayListUnmanaged; const DiffList = ArrayListUnmanaged(Diff); const PatchList = ArrayListUnmanaged(Patch); +//| XXX This boolean is entirely for calming the compiler down while working + +const XXX = false; + //| Fields /// Number of milliseconds to map a diff before giving up (0 for infinity). @@ -1848,9 +1852,8 @@ pub fn matchMain(allocator: Allocator, text: []const u8, pattern: []const u8, pa return loc; } else { // Do a fuzzy compare. - // return match_bitap(allocator, text, pattern, loc); + return matchBitap(allocator, text, pattern, loc); } - _ = allocator; } /// Locate the best instance of 'pattern' in 'text' near 'loc' using the @@ -1868,6 +1871,7 @@ fn matchBitap( // TODO decide what to do here: // assert (Match_MaxBits == 0 || pattern.Length <= Match_MaxBits) // : "Pattern too long for this application."; + // Initialise the alphabet. var map = try matchAlphabet(allocator, pattern); defer map.deinit(); @@ -1984,6 +1988,7 @@ fn matchBitapScore(e: usize, x: usize, loc: usize, pattern: []const u8) f64 { /// @return Hash of character locations. fn matchAlphabet(allocator: Allocator, pattern: []const u8) !std.HashMap(u8, usize) { var map = std.HashMap(u8, usize).init(allocator); + errdefer map.deinit(); for (pattern) |c| { if (!map.contains(c)) { try map.put(c, 0); @@ -2008,6 +2013,7 @@ fn matchAlphabet(allocator: Allocator, pattern: []const u8) !std.HashMap(u8, usi fn patchAddContext(allocator: Allocator, patch: *Patch, text: []const u8) !void { if (text.len == 0) return; // TODO the fixup logic here might make patterns too large? + // It should be ok, because big patches get broken up. Hmm. var padding = 0; { // Grow the pattern around the patch until unique, to set padding amount. var pattern = text[patch.start2 .. patch.start2 + patch.length1]; @@ -2198,12 +2204,216 @@ pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !Patch // TODO other makePatch methods... +/// Merge a set of patches onto the text. Returns a tuple: the first of which +/// is the patched text, the second of which is a PatchList, which may be empty, +/// containing patches which were not successfully applied. +/// +/// TODO I'm just going to return a boolean saying whether all patches +/// were successful. Rethink this at some point. +/// +/// @param patches Array of Patch objects +/// @param text Old text. +/// @return Two element Object array, containing the new text and an array of +/// bool values. +pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const u8) !struct { []const u8, bool } { + if (og_patches.items.len == 0) { + // As silly as this is, we dupe the text, because something + // passing an empty patchset isn't going to check, and will + // end up double-freeing if we don't. Going with 'true' as + // the null patchset was successfully 'applied' here. + return .{ try allocator.dupe(u8, og_text), true }; + } + // Deep copy the patches so that no changes are made to originals. + const patches = try patchListClone(allocator, og_patches); + defer patches.deinit(allocator); + const null_padding = try patchAddPadding(patches); + const text = try std.mem.concat( + u8, + .{ + null_padding, + og_text, + null_padding, + }, + ); + // XXX try patchSplitMax(allocator, patches); + var x: usize = 0; + // delta keeps track of the offset between the expected and actual + // location of the previous patch. If there are patches expected at + // positions 10 and 20, but the first patch was found at 12, delta is 2 + // and the second patch has an effective expected position of 22. + var delta: usize = 0; + for (patches) |a_patch| { + const expected_loc = a_patch.start2 + delta; + const text1 = try diffBeforeText(allocator, a_patch.diffs); + defer allocator.free(text1); + var start_loc: ?usize = null; + var end_loc: ?usize = null; + const m_max_b = @This().match_max_bits; + if (text1.len > m_max_b) { + // patch_splitMax will only provide an oversized pattern + // in the case of a monster delete. + start_loc = matchMain(allocator, text[0..m_max_b], expected_loc); + if (start_loc) |start| { + const e_start = text1.len - m_max_b; + end_loc = matchMain(allocator, text1[e_start .. e_start + expected_loc]); + if (end_loc) |end| { + // + } + } + // end_loc = match_main(text, + // text1.Substring(text1.Length - this.Match_MaxBits), + // expected_loc + text1.Length - this.Match_MaxBits); + // if (end_loc == -1 || start_loc >= end_loc) { + // // Can't find valid trailing context. Drop this patch. + // start_loc = -1; + // } + // } + } + } +} + +// if (text1.Length > this.Match_MaxBits) { + +// } else { +// start_loc = this.match_main(text, text1, expected_loc); +// } +// if (start_loc == -1) { +// // No match found. :( +// results[x] = false; +// // Subtract the delta for this failed patch from subsequent patches. +// delta -= aPatch.length2 - aPatch.length1; +// } else { +// // Found a match. :) +// results[x] = true; +// delta = start_loc - expected_loc; +// string text2; +// if (end_loc == -1) { +// text2 = text.JavaSubstring(start_loc, +// Math.Min(start_loc + text1.Length, text.Length)); +// } else { +// text2 = text.JavaSubstring(start_loc, +// Math.Min(end_loc + this.Match_MaxBits, text.Length)); +// } +// if (text1 == text2) { +// // Perfect match, just shove the Replacement text in. +// text = text.Substring(0, start_loc) + diff_text2(aPatch.diffs) +// + text.Substring(start_loc + text1.Length); +// } else { +// // Imperfect match. Run a diff to get a framework of equivalent +// // indices. +// List diffs = diff_main(text1, text2, false); +// if (text1.Length > this.Match_MaxBits +// && this.diff_levenshtein(diffs) / (float) text1.Length +// > this.Patch_DeleteThreshold) { +// // The end points match, but the content is unacceptably bad. +// results[x] = false; +// } else { +// diff_cleanupSemanticLossless(diffs); +// int index1 = 0; +// foreach (Diff aDiff in aPatch.diffs) { +// if (aDiff.operation != Operation.EQUAL) { +// int index2 = diff_xIndex(diffs, index1); +// if (aDiff.operation == Operation.INSERT) { +// // Insertion +// text = text.Insert(start_loc + index2, aDiff.text); +// } else if (aDiff.operation == Operation.DELETE) { +// // Deletion +// text = text.Remove(start_loc + index2, diff_xIndex(diffs, +// index1 + aDiff.text.Length) - index2); +// } +// } +// if (aDiff.operation != Operation.DELETE) { +// index1 += aDiff.text.Length; +// } +// } +// } +// } +// } +// x++; +// } +// // Strip the padding off. +// text = text.Substring(nullPadding.Length, text.Length +// - 2 * nullPadding.Length); +// return new Object[] { text, results }; +// + +/// Add some padding on text start and end so that edges can match something. +/// Intended to be called only from within patch_apply. +/// @param patches Array of Patch objects. +/// @return The padding string added to each side. +fn patchAddPadding(allocator: Allocator, patches: PatchList) ![]const u8 { + // + if (XXX) { + _ = allocator; + _ = patches; + } +} +// public string patch_addPadding(List patches) { +// short paddingLength = this.Patch_Margin; +// string nullPadding = string.Empty; +// for (short x = 1; x <= paddingLength; x++) { +// nullPadding += (char)x; +// } +// +// // Bump all the patches forward. +// foreach (Patch aPatch in patches) { +// aPatch.start1 += paddingLength; +// aPatch.start2 += paddingLength; +// } +// +// // Add some padding on start of first diff. +// Patch patch = patches.First(); +// List diffs = patch.diffs; +// if (diffs.Count == 0 || diffs.First().operation != Operation.EQUAL) { +// // Add nullPadding equality. +// diffs.Insert(0, new Diff(Operation.EQUAL, nullPadding)); +// patch.start1 -= paddingLength; // Should be 0. +// patch.start2 -= paddingLength; // Should be 0. +// patch.length1 += paddingLength; +// patch.length2 += paddingLength; +// } else if (paddingLength > diffs.First().text.Length) { +// // Grow first equality. +// Diff firstDiff = diffs.First(); +// int extraLength = paddingLength - firstDiff.text.Length; +// firstDiff.text = nullPadding.Substring(firstDiff.text.Length) +// + firstDiff.text; +// patch.start1 -= extraLength; +// patch.start2 -= extraLength; +// patch.length1 += extraLength; +// patch.length2 += extraLength; +// } +// +// // Add some padding on end of last diff. +// patch = patches.Last(); +// diffs = patch.diffs; +// if (diffs.Count == 0 || diffs.Last().operation != Operation.EQUAL) { +// // Add nullPadding equality. +// diffs.Add(new Diff(Operation.EQUAL, nullPadding)); +// patch.length1 += paddingLength; +// patch.length2 += paddingLength; +// } else if (paddingLength > diffs.Last().text.Length) { +// // Grow last equality. +// Diff lastDiff = diffs.Last(); +// int extraLength = paddingLength - lastDiff.text.Length; +// lastDiff.text += nullPadding.Substring(0, extraLength); +// patch.length1 += extraLength; +// patch.length2 += extraLength; +// } +// +// return nullPadding; +// } + /// /// Given an array of patches, return another array that is identical. /// @param patches Array of Patch objects. /// @return Array of Patch objects. fn patchListClone(allocator: Allocator, patches: PatchList) !PatchList { var new_patches = PatchList{}; + errdefer { + for (new_patches) |p| { + p.deinit(allocator); + } + } new_patches.initCapacity(allocator, patches.items.len); for (patches) |patch| { try new_patches.append(allocator, try patch.clone(allocator)); From 630bfe68e4aabe2e3ace97f9644d91f222ac87b9 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 4 Jul 2024 22:54:36 -0400 Subject: [PATCH 047/176] Rest of patchApply --- DiffMatchPatch.zig | 202 ++++++++++++++++++++++++--------------------- 1 file changed, 109 insertions(+), 93 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 0c9be66..8e0848f 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -30,7 +30,7 @@ match_threshold: f32 = 0.05, match_distance: u32 = 1000, /// The number of bits in a usize. -match_max_bits: u8 = 64, +match_max_bits: u8 = @bitSizeOf(usize), /// When deleting a large block of text (over ~64 characters), how close /// do the contents have to be to match the expected contents. (0.0 = @@ -1856,8 +1856,16 @@ pub fn matchMain(allocator: Allocator, text: []const u8, pattern: []const u8, pa } } -/// Locate the best instance of 'pattern' in 'text' near 'loc' using the +// TODO doubling the bits to fit in usize is nice and all, but there's no +// reason to be limited to that, we have bitsets which can be as large as +// we'd like. This could be passed a comptime power-of-two size, and use +// that to make an ArrayBitSet specialized for several sizes, up to, IDK, +// 2k? Then split very large patches only. 64, 256, 512, 1024, 2028, is +// a nice balance between code size and versatility. + +/// Locate the best instance of `pattern` in `text` near `loc` using the /// Bitap algorithm. Returns -1 if no match found. +/// /// @param text The text to search. /// @param pattern The pattern to search for. /// @param loc The location to search around. @@ -2111,7 +2119,6 @@ fn makePatchInternal( var patch = Patch{}; var char_count1 = 0; var char_count2 = 0; - // This avoids freeing the original copy of the text: var first_patch = true; var prepatch_text = text; @@ -2223,20 +2230,19 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const // the null patchset was successfully 'applied' here. return .{ try allocator.dupe(u8, og_text), true }; } + // So we can report if all patches were applied: + var all_applied = true; // Deep copy the patches so that no changes are made to originals. const patches = try patchListClone(allocator, og_patches); defer patches.deinit(allocator); const null_padding = try patchAddPadding(patches); - const text = try std.mem.concat( - u8, - .{ - null_padding, - og_text, - null_padding, - }, - ); + var text_array = try std.ArrayList(u8).initCapacity(og_text.len); + defer text_array.deinit(); + text_array.appendSlice(null_padding); + text_array.appendSlice(og_text); + text_array.appendSlice(null_padding); // XXX try patchSplitMax(allocator, patches); - var x: usize = 0; + // // delta keeps track of the offset between the expected and actual // location of the previous patch. If there are patches expected at // positions 10 and 20, but the first patch was found at 12, delta is 2 @@ -2246,97 +2252,107 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const const expected_loc = a_patch.start2 + delta; const text1 = try diffBeforeText(allocator, a_patch.diffs); defer allocator.free(text1); - var start_loc: ?usize = null; - var end_loc: ?usize = null; + var maybe_start: ?usize = null; + var maybe_end: ?usize = null; const m_max_b = @This().match_max_bits; if (text1.len > m_max_b) { - // patch_splitMax will only provide an oversized pattern + // patchSplitMax will only provide an oversized pattern // in the case of a monster delete. - start_loc = matchMain(allocator, text[0..m_max_b], expected_loc); - if (start_loc) |start| { + maybe_start = matchMain( + allocator, + text_array.items, + text1[0..m_max_b], + expected_loc, + ); + if (maybe_start) |start| { const e_start = text1.len - m_max_b; - end_loc = matchMain(allocator, text1[e_start .. e_start + expected_loc]); - if (end_loc) |end| { - // + maybe_end = matchMain( + allocator, + text_array.items, + text1[e_start..], + e_start + expected_loc, + ); + // No match if a) no end_loc or b) the matches cross each other. + if (maybe_end) |end| { + if (start >= end) { + maybe_start = null; + } + } else { + maybe_start = null; + } + } + } else { + maybe_start = matchMain(allocator, og_text, text1, expected_loc); + } + if (maybe_start) |start| { + // Found a match. :) + delta = start - expected_loc; + // results[x] = true; + const text2 = t2: { + if (maybe_end) |end| { + break :t2 og_text[start..@min(end + m_max_b, og_text.len)]; + } else { + break :t2 og_text[start..@min(start + text1.len, og_text.len)]; + } + }; + if (std.mem.eql(u8, text1, text2)) { + // Perfect match, just shove the replacement text in. + const diff_text = try diffAfterText(allocator, a_patch.diffs); + defer allocator.free(diff_text); + try text_array.replaceRange(start, text1.len, diff_text); + } else { + // Imperfect match. Run a diff to get a framework of equivalent + // indices. + const diffs = try diff( + @This(), + allocator, + text1, + text2, + false, + ); + const t1_l_float: f64 = @floatFromInt(text1.len); + const bad_match = diffLevenshtein(diffs) / t1_l_float > @This().patch_delete_threshold; + if (text1.len > m_max_b and bad_match) { + // The end points match, but the content is unacceptably bad. + // results[x] = false; + all_applied = false; + } else { + diffCleanupSemanticLossless(allocator, diffs); + var index1: usize = 0; + for (diffs) |a_diff| { + if (a_diff.operation != .equal) { + const index2 = diffIndex(diffs, index1); + if (a_diff.operation == .insert) { + // Insertion + try text_array.insertSlice(start + index2, a_diff.text); + } else if (a_diff.operation == .delete) { + // Deletion + try text_array.replaceRange( + start + index2, + diffIndex(diffs, index1 + a_diff.text.len), + .{}, + ); + } + if (a_diff.operation != .delete) { + index1 += a_diff.text.len; + } + } + } } } - // end_loc = match_main(text, - // text1.Substring(text1.Length - this.Match_MaxBits), - // expected_loc + text1.Length - this.Match_MaxBits); - // if (end_loc == -1 || start_loc >= end_loc) { - // // Can't find valid trailing context. Drop this patch. - // start_loc = -1; - // } - // } + } else { + // No match found. :( + all_applied = false; + // Subtract the delta for this failed patch from subsequent patches. + delta -= a_patch.length2 - a_patch.length1; } } + // strip padding + try text_array.replaceRange(0, null_padding.len, .{}); + text_array.items.len -= null_padding.len; + return .{ text_array.toOwnedSlice(), all_applied }; } -// if (text1.Length > this.Match_MaxBits) { - -// } else { -// start_loc = this.match_main(text, text1, expected_loc); -// } -// if (start_loc == -1) { -// // No match found. :( -// results[x] = false; -// // Subtract the delta for this failed patch from subsequent patches. -// delta -= aPatch.length2 - aPatch.length1; -// } else { -// // Found a match. :) -// results[x] = true; -// delta = start_loc - expected_loc; -// string text2; -// if (end_loc == -1) { -// text2 = text.JavaSubstring(start_loc, -// Math.Min(start_loc + text1.Length, text.Length)); -// } else { -// text2 = text.JavaSubstring(start_loc, -// Math.Min(end_loc + this.Match_MaxBits, text.Length)); -// } -// if (text1 == text2) { -// // Perfect match, just shove the Replacement text in. -// text = text.Substring(0, start_loc) + diff_text2(aPatch.diffs) -// + text.Substring(start_loc + text1.Length); -// } else { -// // Imperfect match. Run a diff to get a framework of equivalent -// // indices. -// List diffs = diff_main(text1, text2, false); -// if (text1.Length > this.Match_MaxBits -// && this.diff_levenshtein(diffs) / (float) text1.Length -// > this.Patch_DeleteThreshold) { -// // The end points match, but the content is unacceptably bad. -// results[x] = false; -// } else { -// diff_cleanupSemanticLossless(diffs); -// int index1 = 0; -// foreach (Diff aDiff in aPatch.diffs) { -// if (aDiff.operation != Operation.EQUAL) { -// int index2 = diff_xIndex(diffs, index1); -// if (aDiff.operation == Operation.INSERT) { -// // Insertion -// text = text.Insert(start_loc + index2, aDiff.text); -// } else if (aDiff.operation == Operation.DELETE) { -// // Deletion -// text = text.Remove(start_loc + index2, diff_xIndex(diffs, -// index1 + aDiff.text.Length) - index2); -// } -// } -// if (aDiff.operation != Operation.DELETE) { -// index1 += aDiff.text.Length; -// } -// } -// } -// } -// } -// x++; -// } -// // Strip the padding off. -// text = text.Substring(nullPadding.Length, text.Length -// - 2 * nullPadding.Length); -// return new Object[] { text, results }; -// - /// Add some padding on text start and end so that edges can match something. /// Intended to be called only from within patch_apply. /// @param patches Array of Patch objects. From bf1f95a1efba66d127eefca14242157a090c186e Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 5 Jul 2024 15:01:43 -0400 Subject: [PATCH 048/176] Add patch split and patch padding Also has a sketch of an improved bitap which can potentially avoid having to split patches in a great many cases. This finishes the bulk of the work in translation. It remains to add the patch conversion functions, and decide what to do about the to and from delta. They aren't used, and I think the format could be substantively improved. Next up: get on the big monitor and do a proofread pass on all the code added since the last tests, then translate tests from the suite up to a point where this has total line coverage and no leaks. --- DiffMatchPatch.zig | 433 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 372 insertions(+), 61 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 8e0848f..d829d40 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1862,6 +1862,115 @@ pub fn matchMain(allocator: Allocator, text: []const u8, pattern: []const u8, pa // that to make an ArrayBitSet specialized for several sizes, up to, IDK, // 2k? Then split very large patches only. 64, 256, 512, 1024, 2028, is // a nice balance between code size and versatility. +// Something like this: +fn matchBitapImproved( + allocator: Allocator, + text: []const u8, + pattern: []const u8, + loc: usize, + UIntType: type, +) ?usize { + assert(pattern.len < @bitSizeOf(UIntType)); + const ShiftWidth = ShiftSizeForType(UIntType); + // Initialise the alphabet. + var map = try matchAlphabet(allocator, pattern); + defer map.deinit(); + // Highest score beyond which we give up. + var threshold = @This().threshold; + // Is there a nearby exact match? (speedup) + var best_loc = std.mem.indexOfPos(u8, text, pattern); + if (best_loc) |best| { + threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + } + // What about in the other direction? (speedup) + const trunc_text = text[0..@min(loc + pattern.len, text.len)]; + best_loc = std.mem.lastIndexOf(u8, trunc_text, pattern); + if (best_loc) |best| { + threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + } + // Initialise the bit arrays. + const shift: ShiftWidth = @intCast(pattern.len - 1); + // 0 for a match for faster bit twiddles + const matchmask = ~(1 << shift); + best_loc = null; + var bin_min: usize = undefined; + var bin_mid: usize = undefined; + var bin_max = pattern.len + text.len; + // null last_rd to simplying freeing memory + var last_rd = try allocator.alloc(UIntType, 0); + for (0..pattern.len) |d| { + // Scan for the best match; each iteration allows for one more error. + // Run a binary search to determine how far from 'loc' we can stray at + // this error level. + bin_min = 0; + bin_mid = bin_max; + while (bin_min < bin_mid) { + if (matchBitapScore(d, loc + bin_mid, loc, pattern) <= threshold) { + bin_min = bin_mid; + } else { + bin_max = bin_mid; + } + bin_mid = (bin_max - bin_min) / 2 + bin_min; + } + // Use the result from this iteration as the maximum for the next. + bin_max = bin_mid; + var start = @max(1, loc - bin_mid + 1); + const finish = @min(loc + bin_mid, text.len) + pattern.len; + var rd = try allocator.alloc(UIntType, finish + 2); + const dshift: ShiftWidth = @intCast(d); + rd[finish + 1] = 1 << dshift; + var j = finish; + while (j >= start) : (j -= 1) { + const char_match: usize = if (text.len <= j - 1 or !map.contains(text[j - 1])) + // Out of range. + 0 + else + map.get(text[j - 1]); + if (d == 0) { + // First pass: exact match. + rd[j] = ((rd[j + 1] << 1)) & char_match; + } else { + // Subsequent passes: fuzzy match. + rd[j] = ((rd[j + 1] << 1)) & char_match & (((last_rd[j + 1] & last_rd[j]) << 1)) & last_rd[j + 1]; + } + if ((rd[j] & matchmask) != 0) { + const score = matchBitapScore(d, j - 1, loc, pattern); + // This match will almost certainly be better than any existing + // match. But check anyway. + if (score <= threshold) { + // Told you so. + threshold = score; + best_loc = j - 1; + if (best_loc > loc) { + // When passing loc, don't exceed our current distance from loc. + start = @max(1, 2 * loc - best_loc); + } else { + // Already passed loc, downhill from here on in. + break; + } + } + } + } + if (matchBitapScore(d + 1, loc, loc, pattern) > threshold) { + // No hope for a (better) match at greater error levels. + break; + } + allocator.free(last_rd); + last_rd = rd; + } + allocator.free(last_rd); + return best_loc; +} + +fn ShiftSizeForType(T: type) type { + return switch (@typeInfo(T.Int.bits)) { + 64 => u6, + 256 => u8, + 1024 => u9, + 2048 => u10, + else => unreachable, + }; +} /// Locate the best instance of `pattern` in `text` near `loc` using the /// Bitap algorithm. Returns -1 if no match found. @@ -1875,7 +1984,7 @@ fn matchBitap( text: []const u8, pattern: []const u8, loc: usize, -) ?usize { +) !?usize { // TODO decide what to do here: // assert (Match_MaxBits == 0 || pattern.Length <= Match_MaxBits) // : "Pattern too long for this application."; @@ -1890,6 +1999,9 @@ fn matchBitap( if (best_loc) |best| { threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); } + // TODO obviously if we want a speedup here, we do this: + // if (threshold == 0.0) return best_loc; + // We don't have to unwrap best_loc because the retval is ?usize already // What about in the other direction? (speedup) const trunc_text = text[0..@min(loc + pattern.len, text.len)]; best_loc = std.mem.lastIndexOf(u8, trunc_text, pattern); @@ -1976,18 +2088,23 @@ fn matchBitap( /// @param pattern Pattern being sought. /// @return Overall score for match (0.0 = good, 1.0 = bad). fn matchBitapScore(e: usize, x: usize, loc: usize, pattern: []const u8) f64 { + // shortcut? TODO, proof in comments + // if (e == 0 and x == loc) return 0.0; const e_float: f32 = @floatFromInt(e); const len_float: f32 = @floatFromInt(pattern.len); + // if e == 0, accuracy == 0: 0/x = 0 const accuracy = e_float / len_float; + // if loc == x, proximity == 0 const proximity = if (loc >= x) loc - x else x - loc; if (@This().match_distance == 0) { // Dodge divide by zero - if (proximity == 0) + if (proximity == 0) // therefore this returns 0 return accuracy else return 1.0; } const float_match: f64 = @floatFromInt(@This().match_distance); + // or this is 0 + 0/f_m aka 0 return accuracy + (proximity / float_match); } @@ -2010,6 +2127,27 @@ fn matchAlphabet(allocator: Allocator, pattern: []const u8) !std.HashMap(u8, usi return map; } +/// Initialise the alphabet for the Bitap algorithm. +/// @param pattern The text to encode. +/// @return Hash of character locations. +fn matchAlphabetImproved(allocator: Allocator, pattern: []const u8, UIntSize: type) !std.HashMap(u8, usize) { + const ShiftType = ShiftSizeForType(UIntSize); + var map = std.HashMap(u8, usize).init(allocator); + errdefer map.deinit(); + for (pattern) |c| { + if (!map.contains(c)) { + try map.put(c, 0); + } + } + for (pattern, 0..) |c, i| { + const shift: ShiftType = @intCast(pattern.len - i - 1); + // TODO I think we want c_mask & ~ 1 << shift here: + const value: UIntSize = map.get(c) | (1 << shift); + try map.put(c, value); + } + return map; +} + //| PATCH FUNCTIONS /// @@ -2241,7 +2379,7 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const text_array.appendSlice(null_padding); text_array.appendSlice(og_text); text_array.appendSlice(null_padding); - // XXX try patchSplitMax(allocator, patches); + try patchSplitMax(allocator, patches); // // delta keeps track of the offset between the expected and actual // location of the previous patch. If there are patches expected at @@ -2353,71 +2491,244 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const return .{ text_array.toOwnedSlice(), all_applied }; } +// Look through the patches and break up any which are longer than the +// maximum limit of the match algorithm. +// Intended to be called only from within patch_apply. +// @param patches List of Patch objects. +fn patchSplitMax(allocator: Allocator, patches: PatchList) !PatchList { + const patch_size = @This().match_max_bits; + const patch_margin = @This().patch_margin; + const max_patch_len = patch_size - patch_size - patch_margin; + // Mutating an array while iterating it? Sure, lets! + var x = 0; + while (x < patches.len) : (x += 1) { + if (patches[x].length1 <= patch_size) continue; + // We have a big ol' patch. + const bigpatch = patches.orderedRemove(x); + defer bigpatch.deinit(allocator); + // Prevent incrementing past the next patch: + x -= 1; + var start1 = bigpatch.start1; + var start2 = bigpatch.start2; + // start with an empty precontext so that we can deinit consistently + var precontext = try allocator.alloc(u8, 0); + while (bigpatch.diffs.items.len != 0) { + // Create one of several smaller patches. + var patch = Patch{}; + var empty = true; + patch.start1 = start1 - precontext.items.len; + patch.start2 = start2 - precontext.items.len; + if (precontext.len != 0) { + patch.length2 = precontext.length; + patch.length1 = patch.length2; + try patch.diffs.append( + allocator, + Diff{ + .operation = .equal, + .text = precontext.toOwnedSlice(), + }, + ); + } + while (bigpatch.diffs.count != 0 and patch.length1 < max_patch_len) { + const diff_type = bigpatch.diffs[0].operation; + const diff_text = bigpatch.diffs[0].text; + if (diff_type == .insert) { + // Insertions are harmless. + patch.length2 += diff_text.len; + start2 += diff_text.len; + // Move the patch (transfers ownership) + const diff1 = bigpatch.diffs.orderedRemove(0); + patch.diffs.append(diff1); + empty = false; + } else if (cond: { + // zig fmt simply will not line break if clauses :/ + const a = diff_type == .delete; + const b = patch.diffs.items.len == 1; + const c = patch.diffs[0].operation == .equal; + const d = diff_text.len > 2 * patch_size; + break :cond a and b and c and d; + }) { + // This is a large deletion. Let it pass in one chunk. + patch.length1 += diff_text.len; + start1 += diff_text.len; + empty = false; + // Transfer to patch: + const diff1 = bigpatch.diffs.orderedRemove(0); + try patch.diffs.append(allocator, diff1); + } else { + // Deletion or equality. Only take as much as we can stomach. + const text_end = @min(diff_text.len, patch_size - patch.length1 - patch_margin); + const new_diff_text = diff_text[0..text_end]; + patch.length += new_diff_text.len; + start1 += new_diff_text.len; + if (diff_type == .equal) { + patch.length2 += diff_text.len; + start2 += diff_text.len; + } else { + empty = false; + } + // Now check if we did anything. + if (new_diff_text.len == diff_text.len) { + // We can reuse the diff. + const diff1 = bigpatch.diffs.orderedRemove(0); + try patch.diffs.append(allocator, diff1); + } else { + // Free and dupe + const old_diff = bigpatch.diffs[0]; + defer old_diff.deinit(allocator); + bigpatch.diffs[0] = Diff{ + .operation = diff_type, + .text = try allocator.dupe(u8, new_diff_text), + }; + } + } + } + // Compute the head context for the next patch. + const context_len: isize = precontext.len - patch_margin; + allocator.free(precontext); + if (context_len > 0) { + const after_text = try diffAfterText(allocator, patch.diffs); + defer allocator.free(after_text); + precontext = try allocator.dupe(u8, after_text[context_len..]); + } else { + precontext = try allocator.alloc(u8, 0); + } + // Append the end context for this patch. + const post_text = try diffBeforeText(bigpatch.diffs); + const postcontext = post: { + if (post_text.len > patch_margin) { + defer allocator.free(post_text); + break :post post_text[0..patch_margin]; + } else { + break :post post_text; + } + }; + if (postcontext.len != 0) { + patch.length1 += postcontext.len; + patch.length2 += postcontext.len; + const maybe_last_diff = patch.diffs.getLastOrNull(); + if (maybe_last_diff) |last_diff| { + if (last_diff.operation == .equal) { + // free this diff and swap in a new one + defer last_diff.deinit(allocator); + patch.diffs.items.len -= 1; + const new_diff_text = try std.mem.concat( + allocator, + last_diff.text, + postcontext, + ); + try patch.diffs.append( + allocator, + Diff{ .operation = .equal, .text = new_diff_text }, + ); + } + } else { + // New diff from postcontext. + try patch.diffs.append( + allocator, + Diff{ .operation = .equal, .text = postcontext }, + ); + } + } else { + // We didn't allocate memory, but it's polite to free it (?) + allocator.free(postcontext); + } + if (!empty) { + // Insert the next patch + // Goes after x, and we need increment to skip: + x += 1; + try patches.insert(allocator, x, patch); + } + } + // free final precontext + allocator.free(precontext); + } +} + /// Add some padding on text start and end so that edges can match something. /// Intended to be called only from within patch_apply. /// @param patches Array of Patch objects. /// @return The padding string added to each side. fn patchAddPadding(allocator: Allocator, patches: PatchList) ![]const u8 { - // - if (XXX) { - _ = allocator; - _ = patches; + assert(patches.items.len != 0); + const pad_len = @This().patch_margin; + var paddingcodes = try std.ArrayList(u8).initCapacity(allocator, pad_len); + defer paddingcodes.deinit(); + { + var control_code: u8 = 1; + while (control_code <= pad_len) : (control_code += 1) { + try paddingcodes.append(control_code); + } + } + // Bump all the patches forward. + for (patches) |a_patch| { + a_patch.start1 += pad_len; + a_patch.start2 += pad_len; + } + // Add some padding on start of first diff. + var patch = patches.items[0]; + var diffs = patch.diffs; + if (diffs.items.len == 0 or diffs.items[0].operation != .equal) { + // Add nullPadding equality. + try diffs.insert( + allocator, + 0, + Diff{ + .operation = .equal, + .text = try allocator.dupe(u8, paddingcodes.items), + }, + ); + patch.start1 -= pad_len; + // OG code says "Should be 0" but this statement is not justified + assert(patch.start1 == 0); + patch.start2 -= pad_len; + assert(patch.start2 == 0); + patch.length1 += pad_len; + patch.lenght2 += pad_len; + } else if (pad_len > diffs.items[0].text.len) { + // Grow first equality. + var diff1 = diffs.items[0]; + defer allocator.free(diff1.text); + const extra_len = pad_len - diff1.text.len; + diff1.text = try std.mem.concat( + allocator, + paddingcodes.items[diff1.text.len..], + diff1.text, + ); + patch.start1 -= extra_len; + patch.start2 -= extra_len; + patch.length1 += extra_len; + patch.length2 += extra_len; + } + // Add some padding on end of last diff. + patch = patches.getLast(); + diffs = patch.diffs; + if (diffs.items.len == 0 or diffs.getLast().opeation != .equal) { + // Add nullPadding equality. + diffs.append( + allocator, + Diff{ + .operation = .equal, + .text = try allocator.dupe(u8, paddingcodes.items), + }, + ); + patch.length1 += pad_len; + patch.length2 += pad_len; + } else if (pad_len > diffs.getLast().text.len) { + // Grow last equality. + var last_diff = diffs.getLast(); + defer allocator.free(last_diff.text); + const extra_len = pad_len - last_diff.text.len; + last_diff.text = try std.mem.concat( + allocator, + last_diff.text, + paddingcodes[0..extra_len], + ); + patch.length1 += extra_len; + patch.length2 += extra_len; } + return paddingcodes.toOwnedSlice(); } -// public string patch_addPadding(List patches) { -// short paddingLength = this.Patch_Margin; -// string nullPadding = string.Empty; -// for (short x = 1; x <= paddingLength; x++) { -// nullPadding += (char)x; -// } -// -// // Bump all the patches forward. -// foreach (Patch aPatch in patches) { -// aPatch.start1 += paddingLength; -// aPatch.start2 += paddingLength; -// } -// -// // Add some padding on start of first diff. -// Patch patch = patches.First(); -// List diffs = patch.diffs; -// if (diffs.Count == 0 || diffs.First().operation != Operation.EQUAL) { -// // Add nullPadding equality. -// diffs.Insert(0, new Diff(Operation.EQUAL, nullPadding)); -// patch.start1 -= paddingLength; // Should be 0. -// patch.start2 -= paddingLength; // Should be 0. -// patch.length1 += paddingLength; -// patch.length2 += paddingLength; -// } else if (paddingLength > diffs.First().text.Length) { -// // Grow first equality. -// Diff firstDiff = diffs.First(); -// int extraLength = paddingLength - firstDiff.text.Length; -// firstDiff.text = nullPadding.Substring(firstDiff.text.Length) -// + firstDiff.text; -// patch.start1 -= extraLength; -// patch.start2 -= extraLength; -// patch.length1 += extraLength; -// patch.length2 += extraLength; -// } -// -// // Add some padding on end of last diff. -// patch = patches.Last(); -// diffs = patch.diffs; -// if (diffs.Count == 0 || diffs.Last().operation != Operation.EQUAL) { -// // Add nullPadding equality. -// diffs.Add(new Diff(Operation.EQUAL, nullPadding)); -// patch.length1 += paddingLength; -// patch.length2 += paddingLength; -// } else if (paddingLength > diffs.Last().text.Length) { -// // Grow last equality. -// Diff lastDiff = diffs.Last(); -// int extraLength = paddingLength - lastDiff.text.Length; -// lastDiff.text += nullPadding.Substring(0, extraLength); -// patch.length1 += extraLength; -// patch.length2 += extraLength; -// } -// -// return nullPadding; -// } /// /// Given an array of patches, return another array that is identical. From 2d2955d90dd27f46f2630d8ccb86f9ccd64421cc Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 5 Jul 2024 17:07:44 -0400 Subject: [PATCH 049/176] Add patch writer That leaves patch reading, and a whole bunch of testing and cleanup. Then a bit of miscellaneous refactoring and improvement... --- DiffMatchPatch.zig | 156 +++++++++++++++++++++++++++++++++++++++++++++ roadmap.md | 4 +- 2 files changed, 159 insertions(+), 1 deletion(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index d829d40..909bc0c 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -149,6 +149,57 @@ pub const Patch = struct { pub fn deinit(patch: *Patch, allocator: Allocator) void { deinitDiffList(allocator, patch.diffs); } + + /// Emit patch in Unidiff format, as specifified here: + /// https://github.com/google/diff-match-patch/wiki/Unidiff + /// This is similar to GNU Unidiff format, but not identical. + /// Header: @@ -382,8 +481,9 @@ + /// Indices are printed as 1-based, not 0-based. + /// @return The GNU diff string. + pub fn asText(patch: Patch, allocator: Allocator) ![]const u8 { + var text_array = std.ArrayList(u8).init(allocator); + defer text_array.deinit(); + const writer = text_array.writer(); + try patch.writeText(writer, patch); + return text_array.toOwnedSlice(); + } + + const format = std.fmt.format; + + /// Stream textual patch representation to Writer. See `asText` + /// for more information. + pub fn writeText(writer: anytype, patch: Patch) !void { + // Write header. + _ = try writer.write("@@ -"); + // Stream coordinates + if (patch.length1 == 0) { + try format(writer, "{d},0", .{patch.start1}); + } else if (patch.length1 == 1) { + try format(writer, "{d}", .{patch.start1 + 1}); + } else { + try format(writer, "{d},{d}", .{ patch.start1 + 1, patch.length1 }); + } + _ = try writer.write(" +"); + if (patch.length2 == 0) { + try std.fmt.format(writer, "{d},0", .{patch.start2}); + } else if (patch.length2 == 1) { + _ = try format(writer, "{d}", .{patch.start2 + 1}); + } else { + try format(writer, "{d},{d}", .{ patch.start2 + 1, patch.length2 }); + } + _ = writer.write(" @@\n"); + // Escape the body of the patch with %xx notation. + for (patch.diffs) |a_diff| { + switch (a_diff.operation) { + .insert => try writer.writeByte('+'), + .delete => try writer.writeByte('-'), + .equal => try writer.writeByte('='), + } + _ = try writeUriEncoded(writer, diff.text); + } + try writer.writeByte('\n'); + return; + } }; pub const DiffError = error{OutOfMemory}; @@ -2748,6 +2799,111 @@ fn patchListClone(allocator: Allocator, patches: PatchList) !PatchList { return new_patches; } +/// +/// Take a list of patches and return a textual representation. +/// @param patches List of Patch objects. +/// @return Text representation of patches. +/// +pub fn patchToText(allocator: Allocator, patches: PatchList) ![]const u8 { + const text_array = try std.ArrayList(u8).init(allocator); + defer text_array.deinit(); + const writer = text_array.writer(); + try writePatch(writer, patches); + return text_array.toOwnedSlice(); +} + +/// Stream a `PatchList` to the provided Writer. +pub fn writePatch(writer: anytype, patches: PatchList) !void { + for (patches) |a_patch| { + try a_patch.writePatch(writer); + } +} + +// /** +// * Parse a textual representation of patches and return a List of Patch +// * objects. +// * @param textline Text representation of patches. +// * @return List of Patch objects. +// * @throws ArgumentException If invalid input. +// */ +// public List patch_fromText(string textline) { +// List patches = new List(); +// if (textline.Length == 0) { +// return patches; +// } +// string[] text = textline.Split('\n'); +// int textPointer = 0; +// Patch patch; +// Regex patchHeader +// = new Regex("^@@ -(\\d+),?(\\d*) \\+(\\d+),?(\\d*) @@$"); +// Match m; +// char sign; +// string line; +// while (textPointer < text.Length) { +// m = patchHeader.Match(text[textPointer]); +// if (!m.Success) { +// throw new ArgumentException("Invalid patch string: " +// + text[textPointer]); +// } +// patch = new Patch(); +// patches.Add(patch); +// patch.start1 = Convert.ToInt32(m.Groups[1].Value); +// if (m.Groups[2].Length == 0) { +// patch.start1--; +// patch.length1 = 1; +// } else if (m.Groups[2].Value == "0") { +// patch.length1 = 0; +// } else { +// patch.start1--; +// patch.length1 = Convert.ToInt32(m.Groups[2].Value); +// } + +// patch.start2 = Convert.ToInt32(m.Groups[3].Value); +// if (m.Groups[4].Length == 0) { +// patch.start2--; +// patch.length2 = 1; +// } else if (m.Groups[4].Value == "0") { +// patch.length2 = 0; +// } else { +// patch.start2--; +// patch.length2 = Convert.ToInt32(m.Groups[4].Value); +// } +// textPointer++; + +// while (textPointer < text.Length) { +// try { +// sign = text[textPointer][0]; +// } catch (IndexOutOfRangeException) { +// // Blank line? Whatever. +// textPointer++; +// continue; +// } +// line = text[textPointer].Substring(1); +// line = line.Replace("+", "%2b"); +// line = HttpUtility.UrlDecode(line); +// if (sign == '-') { +// // Deletion. +// patch.diffs.Add(new Diff(Operation.DELETE, line)); +// } else if (sign == '+') { +// // Insertion. +// patch.diffs.Add(new Diff(Operation.INSERT, line)); +// } else if (sign == ' ') { +// // Minor equality. +// patch.diffs.Add(new Diff(Operation.EQUAL, line)); +// } else if (sign == '@') { +// // Start of next patch. +// break; +// } else { +// // WTF? +// throw new ArgumentException( +// "Invalid patch mode '" + sign + "' in: " + line); +// } +// textPointer++; +// } +// } +// return patches; +// } + /// /// Borrowed from https://github.com/elerch/aws-sdk-for-zig/blob/master/src/aws_http.zig /// under the MIT license. Thanks! diff --git a/roadmap.md b/roadmap.md index 18b50e3..7261c8a 100644 --- a/roadmap.md +++ b/roadmap.md @@ -1,7 +1,8 @@ # Roadmap - [ ] Port patch -- [ ] Port match +- [✅] Port match +- [ ] Port test coverage - [ ] Diff stream - [ ] Use Unicode characters and codepoint indices - 32 - [ ] Implement line diff as a stream @@ -15,6 +16,7 @@ - [ ] Imara diff has an optimized histogram: https://github.com/pascalkuthe/imara-diff - [ ] POSIX-diff compatible patch output? + - [ ] This one seems pretty worthwhile to me. - [ ] Delta functions? They aren't used internally. Covers the bases. From b703ac66bafcbbef7f8bd605838e97e1bcfb19b6 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sat, 6 Jul 2024 01:36:32 -0400 Subject: [PATCH 050/176] Patch parser complete That's the lot of it. It will take several more days to get it covered and debugged, and I do want to do something about the API. But for the most part, this is feature complete. --- DiffMatchPatch.zig | 243 +++++++++++++++++++++++++++++---------------- 1 file changed, 155 insertions(+), 88 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 909bc0c..1061c9b 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -8,6 +8,11 @@ const ArrayListUnmanaged = std.ArrayListUnmanaged; const DiffList = ArrayListUnmanaged(Diff); const PatchList = ArrayListUnmanaged(Patch); +pub const DiffError = error{ + OutOfMemory, + BadPatchString, +}; + //| XXX This boolean is entirely for calming the compiler down while working const XXX = false; @@ -170,7 +175,7 @@ pub const Patch = struct { /// for more information. pub fn writeText(writer: anytype, patch: Patch) !void { // Write header. - _ = try writer.write("@@ -"); + _ = try writer.write(PATCH_HEAD); // Stream coordinates if (patch.length1 == 0) { try format(writer, "{d},0", .{patch.start1}); @@ -187,7 +192,7 @@ pub const Patch = struct { } else { try format(writer, "{d},{d}", .{ patch.start2 + 1, patch.length2 }); } - _ = writer.write(" @@\n"); + _ = writer.write(PATCH_TAIL); // Escape the body of the patch with %xx notation. for (patch.diffs) |a_diff| { switch (a_diff.operation) { @@ -196,13 +201,14 @@ pub const Patch = struct { .equal => try writer.writeByte('='), } _ = try writeUriEncoded(writer, diff.text); + try writer.writeByte('\n'); } - try writer.writeByte('\n'); return; } }; -pub const DiffError = error{OutOfMemory}; +const PATCH_HEAD = "@@ -"; +const PATCH_TAIL = " @@\n"; /// Find the differences between two texts. /// @param before Old string to be diffed. @@ -2819,90 +2825,151 @@ pub fn writePatch(writer: anytype, patches: PatchList) !void { } } -// /** -// * Parse a textual representation of patches and return a List of Patch -// * objects. -// * @param textline Text representation of patches. -// * @return List of Patch objects. -// * @throws ArgumentException If invalid input. -// */ -// public List patch_fromText(string textline) { -// List patches = new List(); -// if (textline.Length == 0) { -// return patches; -// } -// string[] text = textline.Split('\n'); -// int textPointer = 0; -// Patch patch; -// Regex patchHeader -// = new Regex("^@@ -(\\d+),?(\\d*) \\+(\\d+),?(\\d*) @@$"); -// Match m; -// char sign; -// string line; -// while (textPointer < text.Length) { -// m = patchHeader.Match(text[textPointer]); -// if (!m.Success) { -// throw new ArgumentException("Invalid patch string: " -// + text[textPointer]); -// } -// patch = new Patch(); -// patches.Add(patch); -// patch.start1 = Convert.ToInt32(m.Groups[1].Value); -// if (m.Groups[2].Length == 0) { -// patch.start1--; -// patch.length1 = 1; -// } else if (m.Groups[2].Value == "0") { -// patch.length1 = 0; -// } else { -// patch.start1--; -// patch.length1 = Convert.ToInt32(m.Groups[2].Value); -// } - -// patch.start2 = Convert.ToInt32(m.Groups[3].Value); -// if (m.Groups[4].Length == 0) { -// patch.start2--; -// patch.length2 = 1; -// } else if (m.Groups[4].Value == "0") { -// patch.length2 = 0; -// } else { -// patch.start2--; -// patch.length2 = Convert.ToInt32(m.Groups[4].Value); -// } -// textPointer++; - -// while (textPointer < text.Length) { -// try { -// sign = text[textPointer][0]; -// } catch (IndexOutOfRangeException) { -// // Blank line? Whatever. -// textPointer++; -// continue; -// } -// line = text[textPointer].Substring(1); -// line = line.Replace("+", "%2b"); -// line = HttpUtility.UrlDecode(line); -// if (sign == '-') { -// // Deletion. -// patch.diffs.Add(new Diff(Operation.DELETE, line)); -// } else if (sign == '+') { -// // Insertion. -// patch.diffs.Add(new Diff(Operation.INSERT, line)); -// } else if (sign == ' ') { -// // Minor equality. -// patch.diffs.Add(new Diff(Operation.EQUAL, line)); -// } else if (sign == '@') { -// // Start of next patch. -// break; -// } else { -// // WTF? -// throw new ArgumentException( -// "Invalid patch mode '" + sign + "' in: " + line); -// } -// textPointer++; -// } -// } -// return patches; -// } +/// +/// Parse a textual representation of patches and return a List of Patch +/// objects. +/// @param textline Text representation of patches. +/// @return List of Patch objects. +/// @throws ArgumentException If invalid input. +pub fn patchFromText(allocator: Allocator, text: []const u8) !PatchList { + if (text.len == 0) return PatchList{}; + var patches = PatchList{}; + var cursor = 0; + while (cursor < text.len) { + // TODO catch BadPatchString here and print diagnostic + const cursor_delta, const patch = try patchFromHeader(allocator, text[cursor..]); + cursor += cursor_delta; + try patches.append(allocator, patch); + } +} + +fn countDigits(text: []const u8) usize { + var idx = 0; + while (std.ascii.isDigit(text[idx])) : (idx += 1) {} + return idx; +} + +fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patch } { + var patch = Patch{}; + var cursor: usize = undefined; + if (std.mem.eql(u8, text[0..4], PATCH_HEAD)) { + // Parse location and length in before text + patch.start1 = std.fmt.parseInt( + usize, + text[4..], + 10, + ) catch return error.BadPatchString; + cursor = 4 + countDigits(text[4..]); + assert(cursor > 4); + if (text[cursor] != ',') { + cursor += 1; + patch.start1 -= 1; + patch.length1 = 1; + } else { + cursor += 1; + patch.length1 = std.fmt.parseInt( + usize, + text[cursor..], + 10, + ) catch return error.BadPatchString; + const delta = countDigits(text[cursor..]); + assert(delta > 0); + cursor += delta; + if (patch.length1 != 0) { + patch.start1 -= 1; + } + } + } else return error.BadPatchString; + // Parse location and length in after text. + if (text[cursor] == ' ' and text[cursor + 1] == '+') { + cursor += 2; + patch.start2 = std.fmt.parseInt( + usize, + text[cursor..], + 10, + ) catch return error.BadPatchString; + const delta1 = 4 + countDigits(text[4..]); + assert(delta1 > 0); + cursor += delta1; + if (text[cursor] != ',') { + cursor += 1; + patch.start2 -= 1; + patch.length2 = 1; + } else { + cursor += 1; + patch.length2 = std.fmt.parseInt( + usize, + text[cursor..], + 10, + ) catch return error.BadPatchString; + const delta2 = countDigits(text[cursor..]); + assert(delta2 > 1); + cursor += delta2; + if (patch.length2 != 0) { + patch.start2 -= 1; + } + } + } else return error.BadPatchString; + if (std.mem.eql(u8, text[cursor .. cursor + 4], PATCH_TAIL)) { + cursor += 4; + } else return error.BadPatchString; + // Eat the diffs + const patch_lines = std.mem.splitScalar( + u8, + text[cursor..], + '\n', + ); + // splitScalar means blank lines, but we need that to + // track the cursor + patch_loop: while (patch_lines.next()) |line| { + cursor += line.len + 1; + if (line.len == 0) continue; + // Figure this out TODO + // line = line.Replace("+", "%2b"); + const diff_line = try uriDecode(allocator, line); + switch (line[0]) { + '+' => { // Insertion + try patch.diffs.append( + allocator, + Diff{ + .operation = .insert, + .text = diff_line, + }, + ); + }, + '-' => { // Deletion + try patch.diffs.append( + allocator, + Diff{ + .operation = .delete, + .text = diff_line, + }, + ); + }, + ' ' => { // Minor equality + try patch.diffs.append( + allocator, + Diff{ + .operation = .equal, + .text = diff_line, + }, + ); + }, + '@' => { // Start of next patch + // back out cursor + cursor -= line.len - 1; + break :patch_loop; + }, + else => return error.BadPatchString, + } + } // end while + return .{ cursor, patch }; +} + +fn uriDecode(allocator: Allocator, line: []const u8) ![]const u8 { + // XXX finish the job obvs + return allocator.dupe(u8, line); +} /// /// Borrowed from https://github.com/elerch/aws-sdk-for-zig/blob/master/src/aws_http.zig From 3afffcce39427812dc350678066554336c7a416a Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sat, 6 Jul 2024 23:12:30 -0400 Subject: [PATCH 051/176] Add uriDecode --- DiffMatchPatch.zig | 41 ++++++++++++++++++++++++++++++----------- build.zig | 1 + 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 1061c9b..39157a5 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2407,8 +2407,7 @@ pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !Patch // TODO other makePatch methods... /// Merge a set of patches onto the text. Returns a tuple: the first of which -/// is the patched text, the second of which is a PatchList, which may be empty, -/// containing patches which were not successfully applied. +/// is the patched text, the second of which is... /// /// TODO I'm just going to return a boolean saying whether all patches /// were successful. Rethink this at some point. @@ -2697,7 +2696,7 @@ fn patchSplitMax(allocator: Allocator, patches: PatchList) !PatchList { try patches.insert(allocator, x, patch); } } - // free final precontext + // Free final precontext. allocator.free(precontext); } } @@ -2736,7 +2735,7 @@ fn patchAddPadding(allocator: Allocator, patches: PatchList) ![]const u8 { }, ); patch.start1 -= pad_len; - // OG code says "Should be 0" but this statement is not justified + // OG code says "Should be 0" but this statement is not justified... assert(patch.start1 == 0); patch.start2 -= pad_len; assert(patch.start2 == 0); @@ -2919,14 +2918,14 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc text[cursor..], '\n', ); - // splitScalar means blank lines, but we need that to - // track the cursor + // `splitScalar` means blank lines, but we need that to + // track the cursor. patch_loop: while (patch_lines.next()) |line| { cursor += line.len + 1; if (line.len == 0) continue; - // Figure this out TODO - // line = line.Replace("+", "%2b"); - const diff_line = try uriDecode(allocator, line); + // Microsoft encodes spaces as +, we don't, so we don't need this: + // line = line.Replace("+", "%2b"); + const diff_line = try uriDecode(allocator, line) catch return error.BadPatchString; switch (line[0]) { '+' => { // Insertion try patch.diffs.append( @@ -2966,9 +2965,29 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc return .{ cursor, patch }; } +/// Decode our URI-esque escaping fn uriDecode(allocator: Allocator, line: []const u8) ![]const u8 { - // XXX finish the job obvs - return allocator.dupe(u8, line); + if (std.mem.indexOf(u8, line, '%')) |first| { + // Text to decode. + // Result will always be shorter than line: + var new_line = try std.ArrayList(u8).initCapacity(allocator, line.len); + defer new_line.init; + try new_line.appendSlice(line[0..first]); + var out_buf: [1]u8 = .{0}; + var codeunit = try std.fmt.hexToBytes(&out_buf, line[first + 1 .. first + 3]); + try new_line.append(codeunit[0]); + var cursor = first + 3; + while (std.mem.indexOf(u8, line[cursor..], '%')) |next| { + codeunit = try std.fmt.hexToBytes(&out_buf, line[next + 1 .. next + 3]); + try new_line.append(codeunit[0]); + cursor = next + 3; + } else { + try new_line.appendSlice(line[cursor..]); + } + return new_line.toOwnedSlice(); + } else { + return allocator.dupe(u8, line); + } } /// diff --git a/build.zig b/build.zig index ee140be..0778fe5 100644 --- a/build.zig +++ b/build.zig @@ -30,6 +30,7 @@ pub fn build(b: *std.Build) void { .optimize = optimize, }); const step_tests = b.addRunArtifact(tests); + step_tests.has_side_effects = true; b.step("test", "Run diffz tests").dependOn(&step_tests.step); From f7906a6fe9b9bf88c96043aa5ff641af8a265ce8 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 00:59:32 -0400 Subject: [PATCH 052/176] Add other makePatch interfaces --- DiffMatchPatch.zig | 28 +++++++++++++++++----------- make-file-list.py | 26 ++++++++++++++++++++++++++ roadmap.md | 3 ++- 3 files changed, 45 insertions(+), 12 deletions(-) create mode 100644 make-file-list.py diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 39157a5..735df8e 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2394,7 +2394,6 @@ fn makePatchInternal( } } -/// /// Compute a list of patches to turn text1 into text2. /// text2 is not provided, diffs are the delta between text1 and text2. /// @@ -2404,7 +2403,19 @@ pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !Patch try makePatchInternal(allocator, text, diffs, .copy); } -// TODO other makePatch methods... +pub fn makePatchFromTexts(allocator: Allocator, text1: []const u8, text2: []const u8) !PatchList { + const diffs = try diff(@This(), allocator, text1, text2, true); + if (diffs.items.len > 2) { + try diffCleanupSemantic(diffs); + try diffCleanupEfficiency(diffs); + } + return try makePatchInternal(allocator, text1, diffs, .own); +} + +pub fn makePatchFromDiffs(allocator: Allocator, diffs: DiffList) !PatchList { + const text1 = try diffBeforeText(allocator, diffs); + return try makePatch(allocator, text1, diffs, .copy); +} /// Merge a set of patches onto the text. Returns a tuple: the first of which /// is the patched text, the second of which is... @@ -2436,7 +2447,6 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const text_array.appendSlice(og_text); text_array.appendSlice(null_padding); try patchSplitMax(allocator, patches); - // // delta keeps track of the offset between the expected and actual // location of the previous patch. If there are patches expected at // positions 10 and 20, but the first patch was found at 12, delta is 2 @@ -2549,7 +2559,7 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const // Look through the patches and break up any which are longer than the // maximum limit of the match algorithm. -// Intended to be called only from within patch_apply. +// Intended to be called only from within patchApply. // @param patches List of Patch objects. fn patchSplitMax(allocator: Allocator, patches: PatchList) !PatchList { const patch_size = @This().match_max_bits; @@ -2702,7 +2712,7 @@ fn patchSplitMax(allocator: Allocator, patches: PatchList) !PatchList { } /// Add some padding on text start and end so that edges can match something. -/// Intended to be called only from within patch_apply. +/// Intended to be called only from within patchApply. /// @param patches Array of Patch objects. /// @return The padding string added to each side. fn patchAddPadding(allocator: Allocator, patches: PatchList) ![]const u8 { @@ -2786,7 +2796,6 @@ fn patchAddPadding(allocator: Allocator, patches: PatchList) ![]const u8 { return paddingcodes.toOwnedSlice(); } -/// /// Given an array of patches, return another array that is identical. /// @param patches Array of Patch objects. /// @return Array of Patch objects. @@ -2804,11 +2813,9 @@ fn patchListClone(allocator: Allocator, patches: PatchList) !PatchList { return new_patches; } -/// /// Take a list of patches and return a textual representation. /// @param patches List of Patch objects. /// @return Text representation of patches. -/// pub fn patchToText(allocator: Allocator, patches: PatchList) ![]const u8 { const text_array = try std.ArrayList(u8).init(allocator); defer text_array.deinit(); @@ -2824,7 +2831,6 @@ pub fn writePatch(writer: anytype, patches: PatchList) !void { } } -/// /// Parse a textual representation of patches and return a List of Patch /// objects. /// @param textline Text representation of patches. @@ -2925,7 +2931,7 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc if (line.len == 0) continue; // Microsoft encodes spaces as +, we don't, so we don't need this: // line = line.Replace("+", "%2b"); - const diff_line = try uriDecode(allocator, line) catch return error.BadPatchString; + const diff_line = try decodeUri(allocator, line) catch return error.BadPatchString; switch (line[0]) { '+' => { // Insertion try patch.diffs.append( @@ -2966,7 +2972,7 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc } /// Decode our URI-esque escaping -fn uriDecode(allocator: Allocator, line: []const u8) ![]const u8 { +fn decodeUri(allocator: Allocator, line: []const u8) ![]const u8 { if (std.mem.indexOf(u8, line, '%')) |first| { // Text to decode. // Result will always be shorter than line: diff --git a/make-file-list.py b/make-file-list.py new file mode 100644 index 0000000..33f0df6 --- /dev/null +++ b/make-file-list.py @@ -0,0 +1,26 @@ +import os +import git + +# Variables +REPO_PATH = '/Users/atman/code/opp/ziglibs/diffz' +FILE_NAME = 'DiffMatchPatch.zig' +OUTPUT_DIR = 'file-versions' + +# Initialize the repository +repo = git.Repo(REPO_PATH) + +# Create the output directory if it doesn't exist +if not os.path.exists(OUTPUT_DIR): + os.makedirs(OUTPUT_DIR) + +# Get a list of all commits that modified the file +commits = list(repo.iter_commits(paths=FILE_NAME)) +commits.reverse() + +# Loop through each commit +for i, commit in enumerate(commits): + # Checkout the file from the specific commit + file_content = (repo.git.show(f'{commit.hexsha}:{FILE_NAME}')) + # Write the file content to the output directory with a suffix + with open(os.path.join(OUTPUT_DIR, f'file-{i+1:02d}.zig'), 'w') as f: + f.write(file_content) \ No newline at end of file diff --git a/roadmap.md b/roadmap.md index 7261c8a..7c31c46 100644 --- a/roadmap.md +++ b/roadmap.md @@ -1,6 +1,7 @@ # Roadmap -- [ ] Port patch +- [✅] Port patch + - [ ] Add DiffMatchPatch object instead of @This() (which won't work) - [✅] Port match - [ ] Port test coverage - [ ] Diff stream From 0655ece7cfa59ab280c71dbb3a081ff6f8ea8af3 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 03:13:18 -1000 Subject: [PATCH 053/176] Remove conditional free for zero-length text Co-authored-by: Techatrix --- DiffMatchPatch.zig | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 6350a5f..aeb86be 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -11,9 +11,7 @@ const DiffList = ArrayListUnmanaged(Diff); pub fn deinitDiffList(allocator: Allocator, diffs: *DiffList) void { defer diffs.deinit(allocator); for (diffs.items) |d| { - if (d.text.len > 0) { - allocator.free(d.text); - } + allocator.free(d.text); } } From f044a8ba6df47ac028d6a702ee598c28adc89dca Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 03:16:40 -1000 Subject: [PATCH 054/176] Use explicit directory names in .gitignore Co-authored-by: Techatrix --- .gitignore | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 68557b5..8c9d17e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ -zig-* -.zig-* +.zig-cache +zig-cache +zig-out From 9bb28191ae739268262c24e61ea934eae270ec0c Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 03:22:43 -1000 Subject: [PATCH 055/176] Update build.zig to remove empty library Library steps are only for extern symbols, since DiffMatchPatch (currently) provides no C API, this stage produces an empty static library. Co-authored-by: Techatrix --- build.zig | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/build.zig b/build.zig index dd40eb6..dedeff9 100644 --- a/build.zig +++ b/build.zig @@ -10,18 +10,6 @@ pub fn build(b: *std.Build) void { .optimize = optimize, }); - const lib = b.addStaticLibrary(.{ - .name = "diffz", - .root_source_file = b.path("DiffMatchPatch.zig"), - .target = target, - .optimize = optimize, - }); - - // This declares intent for the library to be installed into the standard - // location when the user invokes the "install" step (the default step when - // running `zig build`). - b.installArtifact(lib); - // Run tests const tests = b.addTest(.{ .name = "tests", From cc640a35442ece5c7b5630573c130eae45788473 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 10:18:33 -0400 Subject: [PATCH 056/176] Free resources when allocation fails This is an edit pass to try and catch every case (outside of tests) where an allocation failure would produce a memory leak. --- DiffMatchPatch.zig | 100 ++++++++++++++++++++++++++++----------------- 1 file changed, 63 insertions(+), 37 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 6350a5f..6d41786 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -122,7 +122,7 @@ fn diffInternal( if (std.mem.eql(u8, before, after)) { var diffs = DiffList{}; if (before.len != 0) { - try diffs.append(allocator, Diff.init(.equal, try allocator.dupe(u8, before))); + try diffsAppend(allocator, &diffs, .equal, before); } return diffs; } @@ -141,13 +141,14 @@ fn diffInternal( // Compute the diff on the middle block. var diffs = try dmp.diffCompute(allocator, trimmed_before, trimmed_after, check_lines, deadline); + errdefer deinitDiffList(allocator, &diffs); // Restore the prefix and suffix. if (common_prefix.len != 0) { - try diffs.insert(allocator, 0, Diff.init(.equal, try allocator.dupe(u8, common_prefix))); + try diffsInsert(allocator, &diffs, 0, .equal, common_prefix); } if (common_suffix.len != 0) { - try diffs.append(allocator, Diff.init(.equal, try allocator.dupe(u8, common_suffix))); + try diffsAppend(allocator, &diffs, .equal, common_suffix); } try diffCleanupMerge(allocator, &diffs); @@ -198,16 +199,17 @@ fn diffCompute( deadline: u64, ) DiffError!DiffList { var diffs = DiffList{}; + errdefer deinitDiffList(allocator, &diffs); if (before.len == 0) { // Just add some text (speedup). - try diffs.append(allocator, Diff.init(.insert, try allocator.dupe(u8, after))); + try diffsAppend(allocator, &diffs, .insert, after); return diffs; } if (after.len == 0) { // Just delete some text (speedup). - try diffs.append(allocator, Diff.init(.delete, try allocator.dupe(u8, before))); + try diffsAppend(allocator, &diffs, .delete, before); return diffs; } @@ -220,17 +222,17 @@ fn diffCompute( .delete else .insert; - try diffs.append(allocator, Diff.init(op, try allocator.dupe(u8, long_text[0..index]))); - try diffs.append(allocator, Diff.init(.equal, try allocator.dupe(u8, short_text))); - try diffs.append(allocator, Diff.init(op, try allocator.dupe(u8, long_text[index + short_text.len ..]))); + try diffsAppend(allocator, &diffs, op, long_text[0..index]); + try diffsAppend(allocator, &diffs, .equal, short_text); + try diffsAppend(allocator, &diffs, op, long_text[index + short_text.len ..]); return diffs; } if (short_text.len == 1) { // Single character string. // After the previous speedup, the character can't be an equality. - try diffs.append(allocator, Diff.init(.delete, try allocator.dupe(u8, before))); - try diffs.append(allocator, Diff.init(.insert, try allocator.dupe(u8, after))); + try diffsAppend(allocator, &diffs, .delete, before); + try diffsAppend(allocator, &diffs, .insert, after); return diffs; } @@ -257,7 +259,7 @@ fn diffCompute( // Merge the results. diffs = diffs_a; - try diffs.append(allocator, Diff.init(.equal, try allocator.dupe(u8, half_match.common_middle))); + try diffsAppend(allocator, &diffs, .equal, half_match.common_middle); try diffs.appendSlice(allocator, diffs_b.items); return diffs; } @@ -399,11 +401,19 @@ fn diffHalfMatchInternal( } } if (best_common.items.len * 2 >= long_text.len) { + const prefix_before = try allocator.dupe(u8, best_long_text_a); + errdefer allocator.free(prefix_before); + const suffix_before = try allocator.dupe(u8, best_long_text_b); + errdefer allocator.free(suffix_before); + const prefix_after = try allocator.dupe(u8, best_short_text_a); + errdefer allocator.free(prefix_after); + const suffix_after = try allocator.dupe(u8, best_short_text_b); + errdefer allocator.free(suffix_after); return .{ - .prefix_before = try allocator.dupe(u8, best_long_text_a), - .suffix_before = try allocator.dupe(u8, best_long_text_b), - .prefix_after = try allocator.dupe(u8, best_short_text_a), - .suffix_after = try allocator.dupe(u8, best_short_text_b), + .prefix_before = prefix_before, + .suffix_before = suffix_before, + .prefix_after = prefix_after, + .suffix_after = suffix_after, .common_middle = try best_common.toOwnedSlice(allocator), }; } else { @@ -547,8 +557,8 @@ fn diffBisect( // Diff took too long and hit the deadline or // number of diffs equals number of characters, no commonality at all. var diffs = DiffList{}; - try diffs.append(allocator, Diff.init(.delete, try allocator.dupe(u8, before))); - try diffs.append(allocator, Diff.init(.insert, try allocator.dupe(u8, after))); + try diffsAppend(allocator, &diffs, .delete, before); + try diffsAppend(allocator, &diffs, .insert, after); return diffs; } @@ -576,7 +586,9 @@ fn diffBisectSplit( // Compute both diffs serially. var diffs = try dmp.diffInternal(allocator, text1a, text2a, false, deadline); + errdefer deinitDiffList(allocator, &diffs); var diffsb = try dmp.diffInternal(allocator, text1b, text2b, false, deadline); + // Free the list, but not the contents: defer diffsb.deinit(allocator); try diffs.appendSlice(allocator, diffsb.items); @@ -605,7 +617,7 @@ fn diffLineMode( const line_array = a.line_array; var diffs: DiffList = try dmp.diffInternal(allocator, text1, text2, false, deadline); - + errdefer diffs.deinit(allocator); // Convert the diff back to original text. try diffCharsToLines(allocator, diffs.items, line_array.items); // Eliminate freak matches (e.g. blank lines) @@ -828,6 +840,7 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo diffs.items[ii].text = nt; } else { const text = try allocator.dupe(u8, text_insert.items[0..common_length]); + errdefer allocator.free(text); try diffs.insert(allocator, 0, Diff.init(.equal, text)); pointer += 1; } @@ -856,23 +869,18 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo } if (text_delete.items.len != 0) { - try diffs.insert(allocator, pointer, Diff.init( - .delete, - try allocator.dupe(u8, text_delete.items), - )); + try diffsInsert(allocator, diffs, pointer, .delete, text_delete.items); pointer += 1; } if (text_insert.items.len != 0) { - try diffs.insert(allocator, pointer, Diff.init( - .insert, - try allocator.dupe(u8, text_insert.items), - )); + try diffsInsert(allocator, diffs, pointer, .insert, text_insert.items); pointer += 1; } pointer += 1; } else if (pointer != 0 and diffs.items[pointer - 1].operation == .equal) { // Merge this equality with the previous one. // TODO: Fix using realloc or smth + // Note: can't use realloc because the text is const var nt = try allocator.alloc(u8, diffs.items[pointer - 1].text.len + diffs.items[pointer].text.len); const ot = diffs.items[pointer - 1].text; defer (allocator.free(ot)); @@ -991,10 +999,12 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError (last_equality.?.len <= @max(length_insertions2, length_deletions2))) { // Duplicate record. - try diffs.insert( + try diffsInsert( allocator, + diffs, @intCast(equalities.items[equalities.items.len - 1]), - Diff.init(.delete, try allocator.dupe(u8, last_equality.?)), + .delete, + last_equality.?, ); // Change second copy to insert. diffs.items[@intCast(equalities.items[equalities.items.len - 1] + 1)].operation = .insert; @@ -1044,10 +1054,12 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError // Insert an equality and trim the surrounding edits. defer allocator.free(deletion); defer allocator.free(insertion); - try diffs.insert( + try diffsInsert( allocator, + diffs, @intCast(pointer), - Diff.init(.equal, try allocator.dupe(u8, insertion[0..overlap_length1])), + .equal, + insertion[0..overlap_length1], ); diffs.items[@intCast(pointer - 1)].text = try allocator.dupe(u8, deletion[0 .. deletion.len - overlap_length1]); @@ -1063,10 +1075,12 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError // Insert an equality and swap and trim the surrounding edits. defer allocator.free(deletion); defer allocator.free(insertion); - try diffs.insert( + try diffsInsert( allocator, + diffs, @intCast(pointer), - Diff.init(.equal, try allocator.dupe(u8, deletion[0..overlap_length2])), + .equal, + deletion[0..overlap_length2], ); diffs.items[@intCast(pointer - 1)].operation = .insert; const new_minus = try allocator.dupe(u8, insertion[0 .. insertion.len - overlap_length2]); @@ -1307,10 +1321,12 @@ pub fn diffCleanupEfficiency( ((if (pre_ins) 1 else 0) + (if (pre_del) 1 else 0) + (if (post_ins) 1 else 0) + (if (post_del) 1 else 0)) == 3))) { // Duplicate record. - try diffs.insert( + try diffsInsert( allocator, + &diffs, equalities.items[equalities.items.len - 1], - Diff.init(.delete, try allocator.dupe(u8, last_equality)), + .delete, + last_equality, ); // Change second copy to insert. diffs.items[equalities.items[equalities.items.len - 1] + 1].operation = .insert; @@ -1388,10 +1404,20 @@ fn diffCommonOverlap(text1_in: []const u8, text2_in: []const u8) usize { } } +fn diffsAppend(allocator: Allocator, diffs: *DiffList, op: Diff.Operation, text: []const u8) !void { + const new_text = try allocator.dupe(u8, text); + errdefer allocator.free(new_text); + try diffs.append(allocator, Diff{ .operation = op, .text = new_text }); +} + +fn diffsInsert(allocator: Allocator, diffs: *DiffList, index: usize, op: Diff.Operation, text: []const u8) !void { + const new_text = try allocator.dupe(u8, text); + errdefer allocator.free(new_text); + try diffs.insert(allocator, index, Diff{ .operation = op, .text = new_text }); +} + // DONE [✅]: Allocate all text in diffs to -// not cause segfault while freeing; not a problem -// at the moment because we don't free anything :( -// (or was it??) +// not cause segfault while freeing test diffCommonPrefix { // Detect any common suffix. From 4b63907c18bf360d320a2a97efd8f6aadc70380d Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 10:34:45 -0400 Subject: [PATCH 057/176] errdefer freeing slice appends --- DiffMatchPatch.zig | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 38c2aff..150628c 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -254,6 +254,14 @@ fn diffCompute( deadline, ); defer diffs_b.deinit(allocator); + // we have to deinit regardless, so deinitDiffList would be + // a double free: + errdefer { + for (diffs_b.items) |d| { + allocator.free(d.text); + } + } + (deinitDiffList(allocator, diffs_b)); // Merge the results. diffs = diffs_a; @@ -585,11 +593,15 @@ fn diffBisectSplit( // Compute both diffs serially. var diffs = try dmp.diffInternal(allocator, text1a, text2a, false, deadline); errdefer deinitDiffList(allocator, &diffs); - var diffsb = try dmp.diffInternal(allocator, text1b, text2b, false, deadline); + var diffs_b = try dmp.diffInternal(allocator, text1b, text2b, false, deadline); // Free the list, but not the contents: - defer diffsb.deinit(allocator); - - try diffs.appendSlice(allocator, diffsb.items); + defer diffs_b.deinit(allocator); + errdefer { + for (diffs_b.items) |d| { + allocator.free(d.text); + } + } + try diffs.appendSlice(allocator, diffs_b.items); return diffs; } From 83ac54fd0a8d29df61d251748688fc16a8c93509 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 10:37:09 -0400 Subject: [PATCH 058/176] Remove spurious line --- DiffMatchPatch.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 150628c..ab556de 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -261,7 +261,6 @@ fn diffCompute( allocator.free(d.text); } } - (deinitDiffList(allocator, diffs_b)); // Merge the results. diffs = diffs_a; From 7febe83b123db750edabc54f04d0d66d999efde1 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 11:13:45 -0400 Subject: [PATCH 059/176] Merge changes to memory-manage branch --- DiffMatchPatch.zig | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 54eef42..b29ae59 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -3117,18 +3117,6 @@ fn diffsInsert(allocator: Allocator, diffs: *DiffList, index: usize, op: Diff.Op try diffs.insert(allocator, index, Diff{ .operation = op, .text = new_text }); } -fn diffsAppend(allocator: Allocator, diffs: *DiffList, op: Diff.Operation, text: []const u8) !void { - const new_text = try allocator.dupe(u8, text); - errdefer allocator.free(new_text); - try diffs.append(allocator, Diff{ .operation = op, .text = new_text }); -} - -fn diffsInsert(allocator: Allocator, diffs: *DiffList, index: usize, op: Diff.Operation, text: []const u8) !void { - const new_text = try allocator.dupe(u8, text); - errdefer allocator.free(new_text); - try diffs.insert(allocator, index, Diff{ .operation = op, .text = new_text }); -} - // DONE [✅]: Allocate all text in diffs to // not cause segfault while freeing From 5c4847e234d85a8f08d59cd5c42acceffd7707c3 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 11:36:24 -0400 Subject: [PATCH 060/176] Partial transfer of functions to dmp --- DiffMatchPatch.zig | 123 +++++++++++++++++++++++++++++++-------------- 1 file changed, 84 insertions(+), 39 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index b29ae59..f63a65a 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1918,7 +1918,13 @@ pub fn diffLevenshtein(diffs: DiffList) usize { /// @param pattern The pattern to search for. /// @param loc The location to search around. /// @return Best match index or -1. -pub fn matchMain(allocator: Allocator, text: []const u8, pattern: []const u8, passed_loc: usize) ?usize { +pub fn matchMain( + dmp: DiffMatchPatch, + allocator: Allocator, + text: []const u8, + pattern: []const u8, + passed_loc: usize, +) ?usize { // Clamp the loc to fit within text. const loc = @min(passed_loc, text.len); if (std.mem.eql(u8, text, pattern)) { @@ -1933,7 +1939,7 @@ pub fn matchMain(allocator: Allocator, text: []const u8, pattern: []const u8, pa return loc; } else { // Do a fuzzy compare. - return matchBitap(allocator, text, pattern, loc); + return dmp.matchBitap(allocator, text, pattern, loc); } } @@ -1945,6 +1951,7 @@ pub fn matchMain(allocator: Allocator, text: []const u8, pattern: []const u8, pa // a nice balance between code size and versatility. // Something like this: fn matchBitapImproved( + dmp: DiffMatchPatch, allocator: Allocator, text: []const u8, pattern: []const u8, @@ -1957,17 +1964,17 @@ fn matchBitapImproved( var map = try matchAlphabet(allocator, pattern); defer map.deinit(); // Highest score beyond which we give up. - var threshold = @This().threshold; + var threshold = dmp.threshold; // Is there a nearby exact match? (speedup) var best_loc = std.mem.indexOfPos(u8, text, pattern); if (best_loc) |best| { - threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), threshold); } // What about in the other direction? (speedup) const trunc_text = text[0..@min(loc + pattern.len, text.len)]; best_loc = std.mem.lastIndexOf(u8, trunc_text, pattern); if (best_loc) |best| { - threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), threshold); } // Initialise the bit arrays. const shift: ShiftWidth = @intCast(pattern.len - 1); @@ -1986,7 +1993,7 @@ fn matchBitapImproved( bin_min = 0; bin_mid = bin_max; while (bin_min < bin_mid) { - if (matchBitapScore(d, loc + bin_mid, loc, pattern) <= threshold) { + if (dmp.matchBitapScore(d, loc + bin_mid, loc, pattern) <= threshold) { bin_min = bin_mid; } else { bin_max = bin_mid; @@ -2015,7 +2022,7 @@ fn matchBitapImproved( rd[j] = ((rd[j + 1] << 1)) & char_match & (((last_rd[j + 1] & last_rd[j]) << 1)) & last_rd[j + 1]; } if ((rd[j] & matchmask) != 0) { - const score = matchBitapScore(d, j - 1, loc, pattern); + const score = dmp.matchBitapScore(d, j - 1, loc, pattern); // This match will almost certainly be better than any existing // match. But check anyway. if (score <= threshold) { @@ -2032,7 +2039,7 @@ fn matchBitapImproved( } } } - if (matchBitapScore(d + 1, loc, loc, pattern) > threshold) { + if (dmp.matchBitapScore(d + 1, loc, loc, pattern) > threshold) { // No hope for a (better) match at greater error levels. break; } @@ -2061,6 +2068,7 @@ fn ShiftSizeForType(T: type) type { /// @param loc The location to search around. /// @return Best match index or -1. fn matchBitap( + dmp: DiffMatchPatch, allocator: Allocator, text: []const u8, pattern: []const u8, @@ -2074,11 +2082,11 @@ fn matchBitap( var map = try matchAlphabet(allocator, pattern); defer map.deinit(); // Highest score beyond which we give up. - var threshold = @This().threshold; + var threshold = dmp.threshold; // Is there a nearby exact match? (speedup) var best_loc = std.mem.indexOfPos(u8, text, pattern); if (best_loc) |best| { - threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), threshold); } // TODO obviously if we want a speedup here, we do this: // if (threshold == 0.0) return best_loc; @@ -2087,7 +2095,7 @@ fn matchBitap( const trunc_text = text[0..@min(loc + pattern.len, text.len)]; best_loc = std.mem.lastIndexOf(u8, trunc_text, pattern); if (best_loc) |best| { - threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), threshold); } // Initialise the bit arrays. const shift: u6 = @intCast(pattern.len - 1); @@ -2105,7 +2113,7 @@ fn matchBitap( bin_min = 0; bin_mid = bin_max; while (bin_min < bin_mid) { - if (matchBitapScore(d, loc + bin_mid, loc, pattern) <= threshold) { + if (dmp.matchBitapScore(d, loc + bin_mid, loc, pattern) <= threshold) { bin_min = bin_mid; } else { bin_max = bin_mid; @@ -2134,7 +2142,7 @@ fn matchBitap( rd[j] = ((rd[j + 1] << 1) | 1) & char_match | (((last_rd[j + 1] | last_rd[j]) << 1) | 1) | last_rd[j + 1]; } if ((rd[j] & matchmask) != 0) { - const score = matchBitapScore(d, j - 1, loc, pattern); + const score = dmp.matchBitapScore(d, j - 1, loc, pattern); // This match will almost certainly be better than any existing // match. But check anyway. if (score <= threshold) { @@ -2151,7 +2159,7 @@ fn matchBitap( } } } - if (matchBitapScore(d + 1, loc, loc, pattern) > threshold) { + if (dmp.matchBitapScore(d + 1, loc, loc, pattern) > threshold) { // No hope for a (better) match at greater error levels. break; } @@ -2168,7 +2176,13 @@ fn matchBitap( /// @param loc Expected location of match. /// @param pattern Pattern being sought. /// @return Overall score for match (0.0 = good, 1.0 = bad). -fn matchBitapScore(e: usize, x: usize, loc: usize, pattern: []const u8) f64 { +fn matchBitapScore( + dmp: DiffMatchPatch, + e: usize, + x: usize, + loc: usize, + pattern: []const u8, +) f64 { // shortcut? TODO, proof in comments // if (e == 0 and x == loc) return 0.0; const e_float: f32 = @floatFromInt(e); @@ -2177,7 +2191,7 @@ fn matchBitapScore(e: usize, x: usize, loc: usize, pattern: []const u8) f64 { const accuracy = e_float / len_float; // if loc == x, proximity == 0 const proximity = if (loc >= x) loc - x else x - loc; - if (@This().match_distance == 0) { + if (dmp.match_distance == 0) { // Dodge divide by zero if (proximity == 0) // therefore this returns 0 return accuracy @@ -2237,23 +2251,28 @@ fn matchAlphabetImproved(allocator: Allocator, pattern: []const u8, UIntSize: ty /// /// @param patch The patch to grow. /// @param text Source text. -fn patchAddContext(allocator: Allocator, patch: *Patch, text: []const u8) !void { +fn patchAddContext( + dmp: DiffMatchPatch, + allocator: Allocator, + patch: *Patch, + text: []const u8, +) !void { if (text.len == 0) return; // TODO the fixup logic here might make patterns too large? // It should be ok, because big patches get broken up. Hmm. var padding = 0; { // Grow the pattern around the patch until unique, to set padding amount. var pattern = text[patch.start2 .. patch.start2 + patch.length1]; - const max_width: usize = @This().match_max_bits - (2 * @This().patch_margin); + const max_width: usize = dmp.match_max_bits - (2 * dmp.patch_margin); while (std.mem.indexOf(u8, text, pattern) != std.mem.lastIndexOf(u8, text, pattern) and pattern.len < max_width) { - padding += @This().patch_margin; + padding += dmp.patch_margin; const pat_start = @max(0, patch.start2 - padding); const pat_end = pat_start + @min(text.len, patch.start2 + patch.length1 + padding); pattern = text[pat_start..pat_end]; } } // Add one chunk for good luck. - padding += @This().patch_margin; + padding += dmp.patch_margin; // Add the prefix. const prefix = pre: { var pre_start = @max(0, patch.start2 - padding); @@ -2325,6 +2344,7 @@ const DiffHandling = enum { /// @return List of Patch objects. fn makePatchInternal( + dmp: DiffMatchPatch, allocator: Allocator, text: []const u8, diffs: DiffList, @@ -2369,21 +2389,21 @@ fn makePatchInternal( }, .equal => { // - if (a_diff.text.len <= 2 * @This().patch_margin and patch.diffs.items.len != 0 and a_diff != diffs.items[diffs.items.len]) { + if (a_diff.text.len <= 2 * dmp.patch_margin and patch.diffs.items.len != 0 and a_diff != diffs.items[diffs.items.len]) { // Small equality inside a patch. const d = if (diff_act == .copy) a_diff.clone(allocator) else a_diff; try patch.diffs.append(allocator, d); patch.length1 += a_diff.text.len; patch.length2 += a_diff.text.len; } - if (a_diff.text.len >= 2 * @This().patch_margin) { + if (a_diff.text.len >= 2 * dmp.patch_margin) { // Time for a new patch. if (patch.diffs.items.len != 0) { // free the Diff if we own it if (diff_act == .own) { allocator.free(a_diff.text); } - try patchAddContext(allocator, patch, prepatch_text); + try dmp.patchAddContext(allocator, patch, prepatch_text); try patches.append(allocator, patch); patch = Patch{}; // Unlike Unidiff, our patch lists have a rolling context. @@ -2413,7 +2433,7 @@ fn makePatchInternal( // Pick up the leftover patch if not empty. if (patch.diffs.items.len != 0) { - try patchAddContext(allocator, patch, prepatch_text); + try dmp.patchAddContext(allocator, patch, prepatch_text); try patches.append(allocator, patch); } } @@ -2423,22 +2443,36 @@ fn makePatchInternal( /// /// @param text1 Old text. /// @param diffs Array of Diff objects for text1 to text2. -pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !PatchList { - try makePatchInternal(allocator, text, diffs, .copy); +pub fn makePatch( + dmp: DiffMatchPatch, + allocator: Allocator, + text: []const u8, + diffs: DiffList, +) !PatchList { + try dmp.makePatchInternal(allocator, text, diffs, .copy); } -pub fn makePatchFromTexts(allocator: Allocator, text1: []const u8, text2: []const u8) !PatchList { +pub fn makePatchFromTexts( + dmp: DiffMatchPatch, + allocator: Allocator, + text1: []const u8, + text2: []const u8, +) !PatchList { const diffs = try diff(@This(), allocator, text1, text2, true); if (diffs.items.len > 2) { try diffCleanupSemantic(diffs); try diffCleanupEfficiency(diffs); } - return try makePatchInternal(allocator, text1, diffs, .own); + return try dmp.makePatchInternal(allocator, text1, diffs, .own); } -pub fn makePatchFromDiffs(allocator: Allocator, diffs: DiffList) !PatchList { +pub fn makePatchFromDiffs( + dmp: DiffMatchPatch, + allocator: Allocator, + diffs: DiffList, +) !PatchList { const text1 = try diffBeforeText(allocator, diffs); - return try makePatch(allocator, text1, diffs, .copy); + return try dmp.makePatch(allocator, text1, diffs, .copy); } /// Merge a set of patches onto the text. Returns a tuple: the first of which @@ -2451,7 +2485,12 @@ pub fn makePatchFromDiffs(allocator: Allocator, diffs: DiffList) !PatchList { /// @param text Old text. /// @return Two element Object array, containing the new text and an array of /// bool values. -pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const u8) !struct { []const u8, bool } { +pub fn patchApply( + dmp: DiffMatchPatch, + allocator: Allocator, + og_patches: PatchList, + og_text: []const u8, +) !struct { []const u8, bool } { if (og_patches.items.len == 0) { // As silly as this is, we dupe the text, because something // passing an empty patchset isn't going to check, and will @@ -2482,7 +2521,7 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const defer allocator.free(text1); var maybe_start: ?usize = null; var maybe_end: ?usize = null; - const m_max_b = @This().match_max_bits; + const m_max_b = dmp.match_max_bits; if (text1.len > m_max_b) { // patchSplitMax will only provide an oversized pattern // in the case of a monster delete. @@ -2510,7 +2549,7 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const } } } else { - maybe_start = matchMain(allocator, og_text, text1, expected_loc); + maybe_start = dmp.matchMain(allocator, og_text, text1, expected_loc); } if (maybe_start) |start| { // Found a match. :) @@ -2531,8 +2570,7 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const } else { // Imperfect match. Run a diff to get a framework of equivalent // indices. - const diffs = try diff( - @This(), + const diffs = try dmp.diff( allocator, text1, text2, @@ -2585,9 +2623,13 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const // maximum limit of the match algorithm. // Intended to be called only from within patchApply. // @param patches List of Patch objects. -fn patchSplitMax(allocator: Allocator, patches: PatchList) !PatchList { - const patch_size = @This().match_max_bits; - const patch_margin = @This().patch_margin; +fn patchSplitMax( + dmp: DiffMatchPatch, + allocator: Allocator, + patches: PatchList, +) !PatchList { + const patch_size = dmp.match_max_bits; + const patch_margin = dmp.patch_margin; const max_patch_len = patch_size - patch_size - patch_margin; // Mutating an array while iterating it? Sure, lets! var x = 0; @@ -2739,7 +2781,10 @@ fn patchSplitMax(allocator: Allocator, patches: PatchList) !PatchList { /// Intended to be called only from within patchApply. /// @param patches Array of Patch objects. /// @return The padding string added to each side. -fn patchAddPadding(allocator: Allocator, patches: PatchList) ![]const u8 { +fn patchAddPadding( + allocator: Allocator, + patches: PatchList, +) ![]const u8 { assert(patches.items.len != 0); const pad_len = @This().patch_margin; var paddingcodes = try std.ArrayList(u8).initCapacity(allocator, pad_len); From a7ab35c0c0cbb418eb7dc9504c066d0672944cf7 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 11:49:48 -0400 Subject: [PATCH 061/176] Patch tests --- DiffMatchPatch.zig | 1548 ++++++++++++++++++++++++-------------------- 1 file changed, 837 insertions(+), 711 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index ab556de..b273a63 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -58,6 +58,18 @@ pub const Diff = struct { pub fn eql(a: Diff, b: Diff) bool { return a.operation == b.operation and std.mem.eql(u8, a.text, b.text); } + + test eql { + const equal_a: Diff = .{ .operation = .equal, .text = "a" }; + const insert_a: Diff = .{ .operation = .insert, .text = "a" }; + const equal_b: Diff = .{ .operation = .equal, .text = "b" }; + const delete_b: Diff = .{ .operation = .delete, .text = "b" }; + + try testing.expect(equal_a.eql(equal_a)); + try testing.expect(!insert_a.eql(equal_a)); + try testing.expect(!equal_a.eql(equal_b)); + try testing.expect(!equal_a.eql(delete_b)); + } }; /// Number of milliseconds to map a diff before giving up (0 for infinity). @@ -1559,7 +1571,7 @@ test diffHalfMatch { } test diffLinesToChars { - const allocator = std.testing.allocator; + const allocator = testing.allocator; // Convert lines down to characters. var tmp_array_list = std.ArrayList([]const u8).init(allocator); defer tmp_array_list.deinit(); @@ -1633,26 +1645,14 @@ test diffLinesToChars { test diffCharsToLines { const allocator = std.testing.allocator; - const equal_a = Diff.init(.equal, try allocator.dupe(u8, "a")); - defer allocator.free(equal_a.text); - const insert_a = Diff.init(.insert, try allocator.dupe(u8, "a")); - defer allocator.free(insert_a.text); - const equal_b = Diff.init(.equal, try allocator.dupe(u8, "b")); - defer allocator.free(equal_b.text); - const delete_b = Diff.init(.delete, try allocator.dupe(u8, "b")); - defer allocator.free(delete_b.text); - try testing.expect(equal_a.eql(equal_a)); - try testing.expect(!insert_a.eql(equal_a)); - try testing.expect(!equal_a.eql(equal_b)); - try testing.expect(!equal_a.eql(delete_b)); // Convert chars up to lines. - var diffs = DiffList{}; + var diffs = try DiffList.initCapacity(allocator, 2); defer deinitDiffList(allocator, &diffs); - try diffs.appendSlice(allocator, &.{ - Diff{ .operation = .equal, .text = try allocator.dupe(u8, "\u{0001}\u{0002}\u{0001}") }, - Diff{ .operation = .insert, .text = try allocator.dupe(u8, "\u{0002}\u{0001}\u{0002}") }, - }); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "\u{0001}\u{0002}\u{0001}") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "\u{0002}\u{0001}\u{0002}") }); + var tmp_vector = std.ArrayList([]const u8).init(allocator); defer tmp_vector.deinit(); try tmp_vector.append(""); @@ -1660,437 +1660,385 @@ test diffCharsToLines { try tmp_vector.append("beta\n"); try diffCharsToLines(allocator, diffs.items, tmp_vector.items); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - Diff.init(.equal, "alpha\nbeta\nalpha\n"), - Diff.init(.insert, "beta\nalpha\nbeta\n"), - }), diffs.items); + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .equal, .text = "alpha\nbeta\nalpha\n" }, + .{ .operation = .insert, .text = "beta\nalpha\nbeta\n" }, + }, diffs.items); // TODO: Implement exhaustive tests } test diffCleanupMerge { - const allocator = std.testing.allocator; + const allocator = testing.allocator; // Cleanup a messy diff. - var diffs = DiffList{}; - defer deinitDiffList(allocator, &diffs); - try testing.expectEqualDeep( - @as([]const Diff, &[0]Diff{}), - diffs.items, - ); // Null case - - try diffs.appendSlice(allocator, &[_]Diff{ - .{ - .operation = .equal, - .text = try allocator.dupe(u8, "a"), - }, - .{ - .operation = .delete, - .text = try allocator.dupe(u8, "b"), - }, - .{ - .operation = .insert, - .text = try allocator.dupe(u8, "c"), - }, - }); - try diffCleanupMerge(allocator, &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "b" }, .{ .operation = .insert, .text = "c" } }), diffs.items); // No change case - - var diffs2 = DiffList{}; - defer deinitDiffList(allocator, &diffs2); - try diffs2.appendSlice(allocator, &[_]Diff{ - .{ - .operation = .equal, - .text = try allocator.dupe(u8, "a"), - }, - .{ - .operation = .equal, - .text = try allocator.dupe(u8, "b"), - }, - .{ - .operation = .equal, - .text = try allocator.dupe(u8, "c"), - }, - }); - try diffCleanupMerge(allocator, &diffs2); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .equal, .text = "abc" }, - }), diffs2.items); // Merge equalities - - var diffs3 = DiffList{}; - defer deinitDiffList(allocator, &diffs3); - try diffs3.appendSlice(allocator, &[_]Diff{ - .{ - .operation = .delete, - .text = try allocator.dupe(u8, "a"), - }, - .{ - .operation = .delete, - .text = try allocator.dupe(u8, "b"), - }, - .{ - .operation = .delete, - .text = try allocator.dupe(u8, "c"), - }, - }); - try diffCleanupMerge(allocator, &diffs3); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .delete, .text = "abc" }, - }), diffs3.items); // Merge deletions - - var diffs4 = DiffList{}; - defer deinitDiffList(allocator, &diffs4); - try diffs4.appendSlice(allocator, &[_]Diff{ - .{ - .operation = .insert, - .text = try allocator.dupe(u8, "a"), - }, - .{ - .operation = .insert, - .text = try allocator.dupe(u8, "b"), - }, - .{ - .operation = .insert, - .text = try allocator.dupe(u8, "c"), - }, - }); - try diffCleanupMerge(allocator, &diffs4); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .insert, .text = "abc" }, - }), diffs4.items); // Merge insertions - - var diffs5 = DiffList{}; - defer deinitDiffList(allocator, &diffs5); - try diffs5.appendSlice(allocator, &[_]Diff{ - .{ - .operation = .delete, - .text = try allocator.dupe(u8, "a"), - }, - .{ - .operation = .insert, - .text = try allocator.dupe(u8, "b"), - }, - .{ - .operation = .delete, - .text = try allocator.dupe(u8, "c"), - }, - .{ - .operation = .insert, - .text = try allocator.dupe(u8, "d"), - }, - .{ - .operation = .equal, - .text = try allocator.dupe(u8, "e"), - }, - .{ - .operation = .equal, - .text = try allocator.dupe(u8, "f"), - }, - }); - try diffCleanupMerge(allocator, &diffs5); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .delete, .text = "ac" }, - .{ .operation = .insert, .text = "bd" }, - .{ .operation = .equal, .text = "ef" }, - }), diffs5.items); // Merge interweave - - var diffs6 = DiffList{}; - defer deinitDiffList(allocator, &diffs6); - try diffs6.appendSlice(allocator, &[_]Diff{ - .{ - .operation = .delete, - .text = try allocator.dupe(u8, "a"), - }, - .{ - .operation = .insert, - .text = try allocator.dupe(u8, "abc"), - }, - .{ - .operation = .delete, - .text = try allocator.dupe(u8, "dc"), - }, - }); - try diffCleanupMerge(allocator, &diffs6); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .equal, .text = "a" }, - .{ .operation = .delete, .text = "d" }, - .{ .operation = .insert, .text = "b" }, - .{ .operation = .equal, .text = "c" }, - }), diffs6.items); // Prefix and suffix detection - - var diffs7 = DiffList{}; - defer deinitDiffList(allocator, &diffs7); - try diffs7.appendSlice(allocator, &[_]Diff{ - .{ - .operation = .equal, - .text = try allocator.dupe(u8, "x"), - }, - .{ - .operation = .delete, - .text = try allocator.dupe(u8, "a"), - }, - .{ - .operation = .insert, - .text = try allocator.dupe(u8, "abc"), - }, - .{ - .operation = .delete, - .text = try allocator.dupe(u8, "dc"), - }, - .{ - .operation = .equal, - .text = try allocator.dupe(u8, "y"), - }, - }); - try diffCleanupMerge(allocator, &diffs7); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .equal, .text = "xa" }, - .{ .operation = .delete, .text = "d" }, - .{ .operation = .insert, .text = "b" }, - .{ .operation = .equal, .text = "cy" }, - }), diffs7.items); // Prefix and suffix detection with equalities - - var diffs8 = DiffList{}; - defer deinitDiffList(allocator, &diffs8); - try diffs8.appendSlice(allocator, &[_]Diff{ - .{ - .operation = .equal, - .text = try allocator.dupe(u8, "a"), - }, - .{ - .operation = .insert, - .text = try allocator.dupe(u8, "ba"), - }, - .{ - .operation = .equal, - .text = try allocator.dupe(u8, "c"), - }, - }); - try diffCleanupMerge(allocator, &diffs8); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .insert, .text = "ab" }, - .{ .operation = .equal, .text = "ac" }, - }), diffs8.items); // Slide edit left - - var diffs9 = DiffList{}; - defer deinitDiffList(allocator, &diffs9); - try diffs9.appendSlice(allocator, &[_]Diff{ - .{ - .operation = .equal, - .text = try allocator.dupe(u8, "c"), - }, - .{ - .operation = .insert, - .text = try allocator.dupe(u8, "ab"), - }, - .{ - .operation = .equal, - .text = try allocator.dupe(u8, "a"), - }, - }); - try diffCleanupMerge(allocator, &diffs9); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - .{ .operation = .equal, .text = "ca" }, - .{ .operation = .insert, .text = "ba" }, - }), diffs9.items); // Slide edit right - - var diffs10 = DiffList{}; - defer deinitDiffList(allocator, &diffs10); - try diffs10.appendSlice(allocator, &[_]Diff{ - Diff.init( - .equal, - try allocator.dupe(u8, "a"), - ), - Diff.init( - .delete, - try allocator.dupe(u8, "b"), - ), - Diff.init( - .equal, - try allocator.dupe(u8, "c"), - ), - Diff.init( - .delete, - try allocator.dupe(u8, "ac"), - ), - Diff.init( - .equal, - try allocator.dupe(u8, "x"), - ), - }); - try diffCleanupMerge(allocator, &diffs10); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - Diff.init(.delete, "abc"), - Diff.init(.equal, "acx"), - }), diffs10.items); // Slide edit left recursive - - var diffs11 = DiffList{}; - defer deinitDiffList(allocator, &diffs11); - try diffs11.appendSlice(allocator, &[_]Diff{ - Diff.init( - .equal, - try allocator.dupe(u8, "x"), - ), - Diff.init( - .delete, - try allocator.dupe(u8, "ca"), - ), - Diff.init( - .equal, - try allocator.dupe(u8, "c"), - ), - Diff.init( - .delete, - try allocator.dupe(u8, "b"), - ), - Diff.init( - .equal, - try allocator.dupe(u8, "a"), - ), - }); - try diffCleanupMerge(allocator, &diffs11); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - Diff.init(.equal, "xca"), - Diff.init(.delete, "cba"), - }), diffs11.items); // Slide edit right recursive - - var diffs12 = DiffList{}; - defer deinitDiffList(allocator, &diffs12); - try diffs12.appendSlice(allocator, &[_]Diff{ - Diff.init( - .delete, - try allocator.dupe(u8, "b"), - ), - Diff.init( - .insert, - try allocator.dupe(u8, "ab"), - ), - Diff.init( - .equal, - try allocator.dupe(u8, "c"), - ), - }); - try diffCleanupMerge(allocator, &diffs12); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - Diff.init(.insert, "a"), - Diff.init(.equal, "bc"), - }), diffs12.items); // Empty merge - - var diffs13 = DiffList{}; - defer deinitDiffList(allocator, &diffs13); - try diffs13.appendSlice(allocator, &[_]Diff{ - Diff.init(.equal, ""), - Diff.init(.insert, try allocator.dupe(u8, "a")), - Diff.init(.equal, try allocator.dupe(u8, "b")), - }); - try diffCleanupMerge(allocator, &diffs13); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ - Diff.init(.insert, "a"), - Diff.init(.equal, "b"), - }), diffs13.items); // Empty equality + { + // No change case + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "b") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "c") }); + + try diffCleanupMerge(allocator, &diffs); + try diffCleanupMerge(allocator, &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "b" }, .{ .operation = .insert, .text = "c" } }), diffs.items); // No change case + try diffCleanupMerge(allocator, &diffs); + try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "b" }, .{ .operation = .insert, .text = "c" } }), diffs.items); // No change case + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .equal, .text = "a" }, + .{ .operation = .delete, .text = "b" }, + .{ .operation = .insert, .text = "c" }, + }, diffs.items); + } + + { + // Merge equalities + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "b") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "c") }); + + try diffCleanupMerge(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .equal, .text = "abc" }, + }, diffs.items); + } + + { + // Merge deletions + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "a") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "b") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "c") }); + + try diffCleanupMerge(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "abc" }, + }, diffs.items); + } + + { + + // Merge insertions + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "a") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "b") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "c") }); + + try diffCleanupMerge(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .insert, .text = "abc" }, + }, diffs.items); + } + + { + // Merge interweave + var diffs = try DiffList.initCapacity(allocator, 6); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "a") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "b") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "c") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "d") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "e") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "f") }); + + try diffCleanupMerge(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "ac" }, + .{ .operation = .insert, .text = "bd" }, + .{ .operation = .equal, .text = "ef" }, + }, diffs.items); + } + + { + // Prefix and suffix detection + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "a") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "abc") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "dc") }); + + try diffCleanupMerge(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .equal, .text = "a" }, + .{ .operation = .delete, .text = "d" }, + .{ .operation = .insert, .text = "b" }, + .{ .operation = .equal, .text = "c" }, + }, diffs.items); + } + + { + // Prefix and suffix detection with equalities + var diffs = try DiffList.initCapacity(allocator, 5); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "x") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "a") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "abc") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "dc") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "y") }); + + try diffCleanupMerge(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .equal, .text = "xa" }, + .{ .operation = .delete, .text = "d" }, + .{ .operation = .insert, .text = "b" }, + .{ .operation = .equal, .text = "cy" }, + }, diffs.items); + } + + { + // Slide edit left + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "ba") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "c") }); + + try diffCleanupMerge(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .insert, .text = "ab" }, + .{ .operation = .equal, .text = "ac" }, + }, diffs.items); + } + + { + + // Slide edit right + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "c") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "ab") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); + + try diffCleanupMerge(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .equal, .text = "ca" }, + .{ .operation = .insert, .text = "ba" }, + }, diffs.items); + } + + { + + // Slide edit left recursive + var diffs = try DiffList.initCapacity(allocator, 5); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "b") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "c") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "ac") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "x") }); + + try diffCleanupMerge(allocator, &diffs); + + try testing.expectEqualDeep(&.{ + Diff.init(.delete, "abc"), + Diff.init(.equal, "acx"), + }, diffs.items); + } + + { + // Slide edit right recursive + var diffs = try DiffList.initCapacity(allocator, 5); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "x") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "ca") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "c") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "b") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); + + try diffCleanupMerge(allocator, &diffs); + + try testing.expectEqualDeep(&.{ + Diff.init(.equal, "xca"), + Diff.init(.delete, "cba"), + }, diffs.items); + } + + { + // Empty merge + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "b") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "ab") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "c") }); + + try diffCleanupMerge(allocator, &diffs); + + try testing.expectEqualDeep(&.{ + Diff.init(.insert, "a"), + Diff.init(.equal, "bc"), + }, diffs.items); + } + + { + // Empty equality + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = "" }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "a") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "b") }); + + try diffCleanupMerge(allocator, &diffs); + + try testing.expectEqualDeep(&.{ + Diff.init(.insert, "a"), + Diff.init(.equal, "b"), + }, diffs.items); + } } test diffCleanupSemanticLossless { - const allocator = std.testing.allocator; - var diffs = DiffList{}; - try diffCleanupSemanticLossless(allocator, &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[0]Diff{}), diffs.items); // Null case - - var diffs2 = DiffList{}; - defer deinitDiffList(allocator, &diffs2); - try diffs2.appendSlice(allocator, &.{ - Diff.init(.equal, try allocator.dupe(u8, "AAA\r\n\r\nBBB")), - Diff.init(.insert, try allocator.dupe(u8, "\r\nDDD\r\n\r\nBBB")), - Diff.init(.equal, try allocator.dupe(u8, "\r\nEEE")), - }); - try diffCleanupSemanticLossless(allocator, &diffs2); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "AAA\r\n\r\n"), - Diff.init(.insert, "BBB\r\nDDD\r\n\r\n"), - Diff.init(.equal, "BBB\r\nEEE"), - }), diffs2.items); - - var diffs3 = DiffList{}; - defer deinitDiffList(allocator, &diffs3); - try diffs3.appendSlice(allocator, &.{ - Diff.init(.equal, try allocator.dupe(u8, "AAA\r\nBBB")), - Diff.init(.insert, try allocator.dupe(u8, " DDD\r\nBBB")), - Diff.init(.equal, try allocator.dupe(u8, " EEE")), - }); - try diffCleanupSemanticLossless(allocator, &diffs3); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "AAA\r\n"), - Diff.init(.insert, "BBB DDD\r\n"), - Diff.init(.equal, "BBB EEE"), - }), diffs3.items); - - var diffs4 = DiffList{}; - defer deinitDiffList(allocator, &diffs4); - try diffs4.appendSlice(allocator, &.{ - Diff.init(.equal, try allocator.dupe(u8, "The c")), - Diff.init(.insert, try allocator.dupe(u8, "ow and the c")), - Diff.init(.equal, try allocator.dupe(u8, "at.")), - }); - try diffCleanupSemanticLossless(allocator, &diffs4); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "The "), - Diff.init(.insert, "cow and the "), - Diff.init(.equal, "cat."), - }), diffs4.items); - - var diffs5 = DiffList{}; - defer deinitDiffList(allocator, &diffs5); - try diffs5.appendSlice(allocator, &.{ - Diff.init(.equal, try allocator.dupe(u8, "The-c")), - Diff.init(.insert, try allocator.dupe(u8, "ow-and-the-c")), - Diff.init(.equal, try allocator.dupe(u8, "at.")), - }); - try diffCleanupSemanticLossless(allocator, &diffs5); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "The-"), - Diff.init(.insert, "cow-and-the-"), - Diff.init(.equal, "cat."), - }), diffs5.items); - - var diffs6 = DiffList{}; - defer deinitDiffList(allocator, &diffs6); - try diffs6.appendSlice(allocator, &.{ - Diff.init(.equal, try allocator.dupe(u8, "a")), - Diff.init(.delete, try allocator.dupe(u8, "a")), - Diff.init(.equal, try allocator.dupe(u8, "ax")), - }); - try diffCleanupSemanticLossless(allocator, &diffs6); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.delete, "a"), - Diff.init(.equal, "aax"), - }), diffs6.items); - - var diffs7 = DiffList{}; - defer deinitDiffList(allocator, &diffs7); - try diffs7.appendSlice(allocator, &.{ - Diff.init(.equal, try allocator.dupe(u8, "xa")), - Diff.init(.delete, try allocator.dupe(u8, "a")), - Diff.init(.equal, try allocator.dupe(u8, "a")), - }); - try diffCleanupSemanticLossless(allocator, &diffs7); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "xaa"), - Diff.init(.delete, "a"), - }), diffs7.items); - - var diffs8 = DiffList{}; - defer deinitDiffList(allocator, &diffs8); - try diffs8.appendSlice(allocator, &.{ - Diff.init(.equal, try allocator.dupe(u8, "The xxx. The ")), - Diff.init(.insert, try allocator.dupe(u8, "zzz. The ")), - Diff.init(.equal, try allocator.dupe(u8, "yyy.")), - }); - try diffCleanupSemanticLossless(allocator, &diffs8); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "The xxx."), - Diff.init(.insert, " The zzz."), - Diff.init(.equal, " The yyy."), - }), diffs8.items); + const allocator = testing.allocator; + + { + // Null case + var diffs: DiffList = .{}; + try diffCleanupSemanticLossless(allocator, &diffs); + try testing.expectEqualDeep(&[_]Diff{}, diffs.items); + } + + { + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "AAA\r\n\r\nBBB") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "\r\nDDD\r\n\r\nBBB") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "\r\nEEE") }); + + try diffCleanupSemanticLossless(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + Diff.init(.equal, "AAA\r\n\r\n"), + Diff.init(.insert, "BBB\r\nDDD\r\n\r\n"), + Diff.init(.equal, "BBB\r\nEEE"), + }, diffs.items); + } + + { + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "AAA\r\nBBB") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, " DDD\r\nBBB") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, " EEE") }); + + try diffCleanupSemanticLossless(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + Diff.init(.equal, "AAA\r\n"), + Diff.init(.insert, "BBB DDD\r\n"), + Diff.init(.equal, "BBB EEE"), + }, diffs.items); + } + + { + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "The c") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "ow and the c") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "at.") }); + + try diffCleanupSemanticLossless(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + Diff.init(.equal, "The "), + Diff.init(.insert, "cow and the "), + Diff.init(.equal, "cat."), + }, diffs.items); + } + + { + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "The-c") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "ow-and-the-c") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "at.") }); + + try diffCleanupSemanticLossless(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + Diff.init(.equal, "The-"), + Diff.init(.insert, "cow-and-the-"), + Diff.init(.equal, "cat."), + }, diffs.items); + } + + { + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "a") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "ax") }); + + try diffCleanupSemanticLossless(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + Diff.init(.delete, "a"), + Diff.init(.equal, "aax"), + }, diffs.items); + } + + { + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "xa") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "a") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); + + try diffCleanupSemanticLossless(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + Diff.init(.equal, "xaa"), + Diff.init(.delete, "a"), + }, diffs.items); + } + + { + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "The xxx. The ") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "zzz. The ") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "yyy.") }); + + try diffCleanupSemanticLossless(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + Diff.init(.equal, "The xxx."), + Diff.init(.insert, " The zzz."), + Diff.init(.equal, " The yyy."), + }, diffs.items); + } } fn rebuildtexts(allocator: std.mem.Allocator, diffs: DiffList) ![2][]const u8 { @@ -2114,331 +2062,509 @@ fn rebuildtexts(allocator: std.mem.Allocator, diffs: DiffList) ![2][]const u8 { } test diffBisect { - const allocator = std.testing.allocator; - // Normal. + const allocator = testing.allocator; + + const this: DiffMatchPatch = .{ .diff_timeout = 0 }; + const a = "cat"; const b = "map"; - // Since the resulting diff hasn't been normalized, it would be ok if - // the insertion and deletion pairs are swapped. - // If the order changes, tweak this test as required. - var diffs = DiffList{}; - defer deinitDiffList(allocator, &diffs); - var this = default; - try diffs.appendSlice(allocator, &.{ - Diff.init(.delete, try allocator.dupe(u8, "c")), - Diff.init(.insert, try allocator.dupe(u8, "m")), - Diff.init(.equal, try allocator.dupe(u8, "a")), - Diff.init(.delete, try allocator.dupe(u8, "t")), - Diff.init(.insert, try allocator.dupe(u8, "p")), - }); - // Travis TODO not sure if maxInt(u64) is correct for DateTime.MaxValue - var diff_bisect = try this.diffBisect( - allocator, - a, - b, - std.math.maxInt(u64), - ); - defer deinitDiffList(allocator, &diff_bisect); - try testing.expectEqualDeep(diffs, diff_bisect); // Normal. - - // Timeout. - var diffs2 = DiffList{}; - defer deinitDiffList(allocator, &diffs2); - try diffs2.appendSlice(allocator, &.{ - Diff.init(.delete, try allocator.dupe(u8, "cat")), - Diff.init(.insert, try allocator.dupe(u8, "map")), - }); - // Travis TODO not sure if 0 is correct for DateTime.MinValue - var diff_bisect2 = try this.diffBisect(allocator, a, b, 0); - defer deinitDiffList(allocator, &diff_bisect2); - try testing.expectEqualDeep(diffs2, diff_bisect2); // Timeout. -} -const talloc = testing.allocator; + { + // Normal. + + // Since the resulting diff hasn't been normalized, it would be ok if + // the insertion and deletion pairs are swapped. + // If the order changes, tweak this test as required. + // Travis TODO not sure if maxInt(u64) is correct for DateTime.MaxValue + var diffs = try this.diffBisect( + allocator, + a, + b, + std.math.maxInt(u64), + ); + defer deinitDiffList(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "c" }, + .{ .operation = .insert, .text = "m" }, + .{ .operation = .equal, .text = "a" }, + .{ .operation = .delete, .text = "t" }, + .{ .operation = .insert, .text = "p" }, + }, diffs.items); + } + + { + // Timeout. + var diffs2 = DiffList{}; + defer deinitDiffList(allocator, &diffs2); + try diffs2.appendSlice(allocator, &.{ + Diff.init(.delete, try allocator.dupe(u8, "cat")), + Diff.init(.insert, try allocator.dupe(u8, "map")), + }); + // Travis TODO not sure if 0 is correct for DateTime.MinValue + var diffs = try this.diffBisect(allocator, a, b, 0); + defer deinitDiffList(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "cat" }, + .{ .operation = .insert, .text = "map" }, + }, diffs.items); + } +} test diff { - var arena = std.heap.ArenaAllocator.init(talloc); - defer arena.deinit(); - const allocator = std.testing.allocator; + const allocator = testing.allocator; - // Perform a trivial diff. - var diffs = DiffList{}; - defer diffs.deinit(arena.allocator()); - var this = DiffMatchPatch{}; - try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "", "", false)).items); // diff: Null case. + const this: DiffMatchPatch = .{ .diff_timeout = 0 }; - // TODO This is the last set of tests using the arena. Someone should - // rewrite them not to do so. -Sam - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{Diff.init(.equal, "abc")}); - try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "abc", "abc", false)).items); // diff: Equality. + { + // diff: Null case. + var diffs = try this.diff(allocator, "", "", false); + defer deinitDiffList(allocator, &diffs); - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ Diff.init(.equal, "ab"), Diff.init(.insert, "123"), Diff.init(.equal, "c") }); - try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "abc", "ab123c", false)).items); // diff: Simple insertion. + try testing.expectEqual(0, diffs.items.len); + } - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ Diff.init(.equal, "a"), Diff.init(.delete, "123"), Diff.init(.equal, "bc") }); - try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "a123bc", "abc", false)).items); // diff: Simple deletion. + { + // diff: Equality. + var diffs = try this.diff(allocator, "abc", "abc", false); + defer deinitDiffList(allocator, &diffs); - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ Diff.init(.equal, "a"), Diff.init(.insert, "123"), Diff.init(.equal, "b"), Diff.init(.insert, "456"), Diff.init(.equal, "c") }); - try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "abc", "a123b456c", false)).items); // diff: Two insertions. + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .equal, .text = "abc" }, + }, diffs.items); + } - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ Diff.init(.equal, "a"), Diff.init(.delete, "123"), Diff.init(.equal, "b"), Diff.init(.delete, "456"), Diff.init(.equal, "c") }); - try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "a123b456c", "abc", false)).items); // diff: Two deletions. + { + // diff: Simple insertion. + var diffs = try this.diff(allocator, "abc", "ab123c", false); + defer deinitDiffList(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .equal, .text = "ab" }, + .{ .operation = .insert, .text = "123" }, + .{ .operation = .equal, .text = "c" }, + }, diffs.items); + } - // Perform a real diff. - // Switch off the timeout. - this.diff_timeout = 0; - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ Diff.init(.delete, "a"), Diff.init(.insert, "b") }); - try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "a", "b", false)).items); // diff: Simple case #1. + { + // diff: Simple deletion. + var diffs = try this.diff(allocator, "a123bc", "abc", false); + defer deinitDiffList(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .equal, .text = "a" }, + .{ .operation = .delete, .text = "123" }, + .{ .operation = .equal, .text = "bc" }, + }, diffs.items); + } - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ Diff.init(.delete, "Apple"), Diff.init(.insert, "Banana"), Diff.init(.equal, "s are a"), Diff.init(.insert, "lso"), Diff.init(.equal, " fruit.") }); - try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "Apples are a fruit.", "Bananas are also fruit.", false)).items); // diff: Simple case #2. + { + // diff: Two insertions. + var diffs = try this.diff(allocator, "abc", "a123b456c", false); + defer deinitDiffList(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .equal, .text = "a" }, + .{ .operation = .insert, .text = "123" }, + .{ .operation = .equal, .text = "b" }, + .{ .operation = .insert, .text = "456" }, + .{ .operation = .equal, .text = "c" }, + }, diffs.items); + } - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ Diff.init(.delete, "a"), Diff.init(.insert, "\u{0680}"), Diff.init(.equal, "x"), Diff.init(.delete, "\t"), Diff.init(.insert, "\x00") }); - try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "ax\t", "\u{0680}x\x00", false)).items); // diff: Simple case #3. + { + // diff: Two deletions. + var diffs = try this.diff(allocator, "a123b456c", "abc", false); + defer deinitDiffList(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .equal, .text = "a" }, + .{ .operation = .delete, .text = "123" }, + .{ .operation = .equal, .text = "b" }, + .{ .operation = .delete, .text = "456" }, + .{ .operation = .equal, .text = "c" }, + }, diffs.items); + } + + // Perform a real diff. + { + // diff: Simple case #1. + var diffs = try this.diff(allocator, "a", "b", false); + defer deinitDiffList(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "a" }, + .{ .operation = .insert, .text = "b" }, + }, diffs.items); + } - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ Diff.init(.delete, "1"), Diff.init(.equal, "a"), Diff.init(.delete, "y"), Diff.init(.equal, "b"), Diff.init(.delete, "2"), Diff.init(.insert, "xab") }); - try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "1ayb2", "abxab", false)).items); // diff: Overlap #1. + { + // diff: Simple case #2. + var diffs = try this.diff(allocator, "Apples are a fruit.", "Bananas are also fruit.", false); + defer deinitDiffList(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "Apple" }, + .{ .operation = .insert, .text = "Banana" }, + .{ .operation = .equal, .text = "s are a" }, + .{ .operation = .insert, .text = "lso" }, + .{ .operation = .equal, .text = " fruit." }, + }, diffs.items); + } - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ Diff.init(.insert, "xaxcx"), Diff.init(.equal, "abc"), Diff.init(.delete, "y") }); - try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "abcy", "xaxcxabc", false)).items); // diff: Overlap #2. + { + // diff: Simple case #3. + var diffs = try this.diff(allocator, "ax\t", "\u{0680}x\x00", false); + defer deinitDiffList(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "a" }, + .{ .operation = .insert, .text = "\u{0680}" }, + .{ .operation = .equal, .text = "x" }, + .{ .operation = .delete, .text = "\t" }, + .{ .operation = .insert, .text = "\x00" }, + }, diffs.items); + } - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ Diff.init(.delete, "ABCD"), Diff.init(.equal, "a"), Diff.init(.delete, "="), Diff.init(.insert, "-"), Diff.init(.equal, "bcd"), Diff.init(.delete, "="), Diff.init(.insert, "-"), Diff.init(.equal, "efghijklmnopqrs"), Diff.init(.delete, "EFGHIJKLMNOefg") }); - try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "ABCDa=bcd=efghijklmnopqrsEFGHIJKLMNOefg", "a-bcd-efghijklmnopqrs", false)).items); // diff: Overlap #3. + { + // diff: Overlap #1. + var diffs = try this.diff(allocator, "1ayb2", "abxab", false); + defer deinitDiffList(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "1" }, + .{ .operation = .equal, .text = "a" }, + .{ .operation = .delete, .text = "y" }, + .{ .operation = .equal, .text = "b" }, + .{ .operation = .delete, .text = "2" }, + .{ .operation = .insert, .text = "xab" }, + }, diffs.items); + } - diffs.items.len = 0; - try diffs.appendSlice(arena.allocator(), &.{ Diff.init(.insert, " "), Diff.init(.equal, "a"), Diff.init(.insert, "nd"), Diff.init(.equal, " [[Pennsylvania]]"), Diff.init(.delete, " and [[New") }); - try testing.expectEqualDeep(diffs.items, (try this.diff(arena.allocator(), "a [[Pennsylvania]] and [[New", " and [[Pennsylvania]]", false)).items); // diff: Large equality. + { + // diff: Overlap #2. + var diffs = try this.diff(allocator, "abcy", "xaxcxabc", false); + defer deinitDiffList(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .insert, .text = "xaxcx" }, + .{ .operation = .equal, .text = "abc" }, + .{ .operation = .delete, .text = "y" }, + }, diffs.items); + } - // end of Arena Zone + { + // diff: Overlap #3. + var diffs = try this.diff(allocator, "ABCDa=bcd=efghijklmnopqrsEFGHIJKLMNOefg", "a-bcd-efghijklmnopqrs", false); + defer deinitDiffList(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "ABCD" }, + .{ .operation = .equal, .text = "a" }, + .{ .operation = .delete, .text = "=" }, + .{ .operation = .insert, .text = "-" }, + .{ .operation = .equal, .text = "bcd" }, + .{ .operation = .delete, .text = "=" }, + .{ .operation = .insert, .text = "-" }, + .{ .operation = .equal, .text = "efghijklmnopqrs" }, + .{ .operation = .delete, .text = "EFGHIJKLMNOefg" }, + }, diffs.items); + } + + { + // diff: Large equality. + var diffs = try this.diff(allocator, "a [[Pennsylvania]] and [[New", " and [[Pennsylvania]]", false); + defer deinitDiffList(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .insert, .text = " " }, + .{ .operation = .equal, .text = "a" }, + .{ .operation = .insert, .text = "nd" }, + .{ .operation = .equal, .text = " [[Pennsylvania]]" }, + .{ .operation = .delete, .text = " and [[New" }, + }, diffs.items); + } - this.diff_timeout = 100; // 100ms // Increase the text lengths by 1024 times to ensure a timeout. { const a = "`Twas brillig, and the slithy toves\nDid gyre and gimble in the wabe:\nAll mimsy were the borogoves,\nAnd the mome raths outgrabe.\n" ** 1024; const b = "I am the very model of a modern major general,\nI've information vegetable, animal, and mineral,\nI know the kings of England, and I quote the fights historical,\nFrom Marathon to Waterloo, in order categorical.\n" ** 1024; + + const with_timout: DiffMatchPatch = .{ + .diff_timeout = 100, // 100ms + }; + const start_time = std.time.milliTimestamp(); - var time_diff = try this.diff(allocator, a, b, false); - defer deinitDiffList(allocator, &time_diff); + { + var time_diff = try with_timout.diff(allocator, a, b, false); + defer deinitDiffList(allocator, &time_diff); + } const end_time = std.time.milliTimestamp(); + // Test that we took at least the timeout period. - try testing.expect(this.diff_timeout <= end_time - start_time); // diff: Timeout min. + try testing.expect(with_timout.diff_timeout <= end_time - start_time); // diff: Timeout min. // Test that we didn't take forever (be forgiving). // Theoretically this test could fail very occasionally if the // OS task swaps or locks up for a second at the wrong moment. - try testing.expect((this.diff_timeout) * 10000 * 2 > end_time - start_time); // diff: Timeout max. - this.diff_timeout = 0; + try testing.expect((with_timout.diff_timeout) * 10000 * 2 > end_time - start_time); // diff: Timeout max. } + { // Test the linemode speedup. // Must be long to pass the 100 char cutoff. const a = "1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n"; const b = "abcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\n"; + var diff_checked = try this.diff(allocator, a, b, true); defer deinitDiffList(allocator, &diff_checked); + var diff_unchecked = try this.diff(allocator, a, b, false); defer deinitDiffList(allocator, &diff_unchecked); - try testing.expectEqualDeep(diff_checked, diff_unchecked); // diff: Simple line-mode. + + try testing.expectEqualDeep(diff_checked.items, diff_unchecked.items); // diff: Simple line-mode. } + { const a = "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"; const b = "abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij"; + var diff_checked = try this.diff(allocator, a, b, true); defer deinitDiffList(allocator, &diff_checked); + var diff_unchecked = try this.diff(allocator, a, b, false); defer deinitDiffList(allocator, &diff_unchecked); - try testing.expectEqualDeep(diff_checked, diff_unchecked); // diff: Single line-mode. - } - const a = "1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n"; - const b = "abcdefghij\n1234567890\n1234567890\n1234567890\nabcdefghij\n1234567890\n1234567890\n1234567890\nabcdefghij\n1234567890\n1234567890\n1234567890\nabcdefghij\n"; - var diffs_linemode = try this.diff(allocator, a, b, true); - defer deinitDiffList(allocator, &diffs_linemode); - const texts_linemode = try rebuildtexts(allocator, diffs_linemode); - defer { - allocator.free(texts_linemode[0]); - allocator.free(texts_linemode[1]); + try testing.expectEqualDeep(diff_checked.items, diff_unchecked.items); // diff: Single line-mode. } - var diffs_textmode = try this.diff(allocator, a, b, false); - defer deinitDiffList(allocator, &diffs_textmode); - const texts_textmode = try rebuildtexts(allocator, diffs_textmode); - defer { - allocator.free(texts_textmode[0]); - allocator.free(texts_textmode[1]); + + { + // diff: Overlap line-mode. + const a = "1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n"; + const b = "abcdefghij\n1234567890\n1234567890\n1234567890\nabcdefghij\n1234567890\n1234567890\n1234567890\nabcdefghij\n1234567890\n1234567890\n1234567890\nabcdefghij\n"; + + var diffs_linemode = try this.diff(allocator, a, b, true); + defer deinitDiffList(allocator, &diffs_linemode); + + const texts_linemode = try rebuildtexts(allocator, diffs_linemode); + defer { + allocator.free(texts_linemode[0]); + allocator.free(texts_linemode[1]); + } + + var diffs_textmode = try this.diff(allocator, a, b, false); + defer deinitDiffList(allocator, &diffs_textmode); + + const texts_textmode = try rebuildtexts(allocator, diffs_textmode); + defer { + allocator.free(texts_textmode[0]); + allocator.free(texts_textmode[1]); + } + + try testing.expectEqualStrings(texts_textmode[0], texts_linemode[0]); + try testing.expectEqualStrings(texts_textmode[1], texts_linemode[1]); } - try testing.expectEqualDeep(texts_textmode, texts_linemode); // diff: Overlap line-mode. } test diffCleanupSemantic { - const alloc = std.testing.allocator; - // Cleanup semantically trivial equalities. - // Null case. - var diffs_empty = DiffList{}; - defer deinitDiffList(alloc, &diffs_empty); - // var this = default; - try diffCleanupSemantic(alloc, &diffs_empty); - try testing.expectEqual(@as(usize, 0), diffs_empty.items.len); // Null case + const allocator = testing.allocator; - var diffs = DiffList{}; - defer deinitDiffList(alloc, &diffs); - diffs.items.len = 0; - try diffs.appendSlice(alloc, &.{ - Diff.init(.delete, try alloc.dupe(u8, "ab")), - Diff.init(.insert, try alloc.dupe(u8, "cd")), - Diff.init(.equal, try alloc.dupe(u8, "12")), - Diff.init(.delete, try alloc.dupe(u8, "e")), - }); - try diffCleanupSemantic(alloc, &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // No elimination #1 - Diff.init(.delete, "ab"), - Diff.init(.insert, "cd"), - Diff.init(.equal, "12"), - Diff.init(.delete, "e"), - }), diffs.items); - - var diffs2 = DiffList{}; - defer deinitDiffList(alloc, &diffs2); - diffs2.items.len = 0; - try diffs2.appendSlice(alloc, &.{ - Diff.init(.delete, try alloc.dupe(u8, "abc")), - Diff.init(.insert, try alloc.dupe(u8, "ABC")), - Diff.init(.equal, try alloc.dupe(u8, "1234")), - Diff.init(.delete, try alloc.dupe(u8, "wxyz")), - }); - try diffCleanupSemantic(alloc, &diffs2); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // No elimination #2 - Diff.init(.delete, "abc"), - Diff.init(.insert, "ABC"), - Diff.init(.equal, "1234"), - Diff.init(.delete, "wxyz"), - }), diffs2.items); - - var diffs3 = DiffList{}; - defer deinitDiffList(alloc, &diffs3); - try diffs3.appendSlice(alloc, &.{ - Diff.init(.delete, try alloc.dupe(u8, "a")), - Diff.init(.equal, try alloc.dupe(u8, "b")), - Diff.init(.delete, try alloc.dupe(u8, "c")), - }); - try diffCleanupSemantic(alloc, &diffs3); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Simple elimination - Diff.init(.delete, "abc"), - Diff.init(.insert, "b"), - }), diffs3.items); - - var diffs4 = DiffList{}; - defer deinitDiffList(alloc, &diffs4); - try diffs4.appendSlice(alloc, &.{ - Diff.init(.delete, try alloc.dupe(u8, "ab")), - Diff.init(.equal, try alloc.dupe(u8, "cd")), - Diff.init(.delete, try alloc.dupe(u8, "e")), - Diff.init(.equal, try alloc.dupe(u8, "f")), - Diff.init(.insert, try alloc.dupe(u8, "g")), - }); - try diffCleanupSemantic(alloc, &diffs4); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Backpass elimination - Diff.init(.delete, "abcdef"), - Diff.init(.insert, "cdfg"), - }), diffs4.items); - - var diffs5 = DiffList{}; - defer deinitDiffList(alloc, &diffs5); - try diffs5.appendSlice(alloc, &.{ - Diff.init(.insert, try alloc.dupe(u8, "1")), - Diff.init(.equal, try alloc.dupe(u8, "A")), - Diff.init(.delete, try alloc.dupe(u8, "B")), - Diff.init(.insert, try alloc.dupe(u8, "2")), - Diff.init(.equal, try alloc.dupe(u8, "_")), - Diff.init(.insert, try alloc.dupe(u8, "1")), - Diff.init(.equal, try alloc.dupe(u8, "A")), - Diff.init(.delete, try alloc.dupe(u8, "B")), - Diff.init(.insert, try alloc.dupe(u8, "2")), - }); - try diffCleanupSemantic(alloc, &diffs5); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Multiple elimination - Diff.init(.delete, "AB_AB"), - Diff.init(.insert, "1A2_1A2"), - }), diffs5.items); - - var diffs6 = DiffList{}; - defer deinitDiffList(alloc, &diffs6); - try diffs6.appendSlice(alloc, &.{ - Diff.init(.equal, try alloc.dupe(u8, "The c")), - Diff.init(.delete, try alloc.dupe(u8, "ow and the c")), - Diff.init(.equal, try alloc.dupe(u8, "at.")), - }); - try diffCleanupSemantic(alloc, &diffs6); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Word boundaries - Diff.init(.equal, "The "), - Diff.init(.delete, "cow and the "), - Diff.init(.equal, "cat."), - }), diffs6.items); - - var diffs7 = DiffList{}; - defer deinitDiffList(alloc, &diffs7); - try diffs7.appendSlice(alloc, &.{ - Diff.init(.delete, try alloc.dupe(u8, "abcxx")), - Diff.init(.insert, try alloc.dupe(u8, "xxdef")), - }); - try diffCleanupSemantic(alloc, &diffs7); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // No overlap elimination - Diff.init(.delete, "abcxx"), - Diff.init(.insert, "xxdef"), - }), diffs7.items); - - var diffs8 = DiffList{}; - defer deinitDiffList(alloc, &diffs8); - try diffs8.appendSlice(alloc, &.{ - Diff.init(.delete, try alloc.dupe(u8, "abcxxx")), - Diff.init(.insert, try alloc.dupe(u8, "xxxdef")), - }); - try diffCleanupSemantic(alloc, &diffs8); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Overlap elimination - Diff.init(.delete, "abc"), - Diff.init(.equal, "xxx"), - Diff.init(.insert, "def"), - }), diffs8.items); - - var diffs9 = DiffList{}; - defer deinitDiffList(alloc, &diffs9); - try diffs9.appendSlice(alloc, &.{ - Diff.init(.delete, try alloc.dupe(u8, "xxxabc")), - Diff.init(.insert, try alloc.dupe(u8, "defxxx")), - }); - try diffCleanupSemantic(alloc, &diffs9); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Reverse overlap elimination - Diff.init(.insert, "def"), - Diff.init(.equal, "xxx"), - Diff.init(.delete, "abc"), - }), diffs9.items); - - var diffs10 = DiffList{}; - defer deinitDiffList(alloc, &diffs10); - try diffs10.appendSlice(alloc, &.{ - Diff.init(.delete, try alloc.dupe(u8, "abcd1212")), - Diff.init(.insert, try alloc.dupe(u8, "1212efghi")), - Diff.init(.equal, try alloc.dupe(u8, "----")), - Diff.init(.delete, try alloc.dupe(u8, "A3")), - Diff.init(.insert, try alloc.dupe(u8, "3BC")), - }); - try diffCleanupSemantic(alloc, &diffs10); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ // Two overlap eliminations - Diff.init(.delete, "abcd"), - Diff.init(.equal, "1212"), - Diff.init(.insert, "efghi"), - Diff.init(.equal, "----"), - Diff.init(.delete, "A"), - Diff.init(.equal, "3"), - Diff.init(.insert, "BC"), - }), diffs10.items); + { + // Null case. + var diffs: DiffList = .{}; + defer deinitDiffList(allocator, &diffs); + try diffCleanupSemantic(allocator, &diffs); + try testing.expectEqual(@as(usize, 0), diffs.items.len); // Null case + } + + { + // No elimination #1 + var diffs = try DiffList.initCapacity(allocator, 4); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "ab") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "cd") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "12") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "e") }); + + try diffCleanupSemantic(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "ab" }, + .{ .operation = .insert, .text = "cd" }, + .{ .operation = .equal, .text = "12" }, + .{ .operation = .delete, .text = "e" }, + }, diffs.items); + } + + { + // No elimination #2 + var diffs = try DiffList.initCapacity(allocator, 4); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "abc") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "ABC") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "1234") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "wxyz") }); + + try diffCleanupSemantic(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "abc" }, + .{ .operation = .insert, .text = "ABC" }, + .{ .operation = .equal, .text = "1234" }, + .{ .operation = .delete, .text = "wxyz" }, + }, diffs.items); + } + + { + // Simple elimination + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "a") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "b") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "c") }); + + try diffCleanupSemantic(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "abc" }, + .{ .operation = .insert, .text = "b" }, + }, diffs.items); + } + + { + // Backpass elimination + var diffs = try DiffList.initCapacity(allocator, 5); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "ab") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "cd") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "e") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "f") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "g") }); + + try diffCleanupSemantic(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "abcdef" }, + .{ .operation = .insert, .text = "cdfg" }, + }, diffs.items); + } + + { + // Multiple elimination + var diffs = try DiffList.initCapacity(allocator, 9); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "1") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "A") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "B") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "2") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "_") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "1") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "A") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "B") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "2") }); + + try diffCleanupSemantic(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "AB_AB" }, + .{ .operation = .insert, .text = "1A2_1A2" }, + }, diffs.items); + } + + { + // Word boundaries + var diffs = try DiffList.initCapacity(allocator, 3); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "The c") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "ow and the c") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "at.") }); + + try diffCleanupSemantic(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .equal, .text = "The " }, + .{ .operation = .delete, .text = "cow and the " }, + .{ .operation = .equal, .text = "cat." }, + }, diffs.items); + } + + { + // No overlap elimination + var diffs = try DiffList.initCapacity(allocator, 2); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "abcxx") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "xxdef") }); + + try diffCleanupSemantic(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "abcxx" }, + .{ .operation = .insert, .text = "xxdef" }, + }, diffs.items); + } + + { + // Overlap elimination + var diffs = try DiffList.initCapacity(allocator, 2); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "abcxxx") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "xxxdef") }); + + try diffCleanupSemantic(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "abc" }, + .{ .operation = .equal, .text = "xxx" }, + .{ .operation = .insert, .text = "def" }, + }, diffs.items); + } + + { + // Reverse overlap elimination + var diffs = try DiffList.initCapacity(allocator, 2); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "xxxabc") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "defxxx") }); + + try diffCleanupSemantic(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .insert, .text = "def" }, + .{ .operation = .equal, .text = "xxx" }, + .{ .operation = .delete, .text = "abc" }, + }, diffs.items); + } + + { + // Two overlap eliminations + var diffs = try DiffList.initCapacity(allocator, 5); + defer deinitDiffList(allocator, &diffs); + + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "abcd1212") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "1212efghi") }); + diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "----") }); + diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "A3") }); + diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "3BC") }); + + try diffCleanupSemantic(allocator, &diffs); + + try testing.expectEqualDeep(&[_]Diff{ + .{ .operation = .delete, .text = "abcd" }, + .{ .operation = .equal, .text = "1212" }, + .{ .operation = .insert, .text = "efghi" }, + .{ .operation = .equal, .text = "----" }, + .{ .operation = .delete, .text = "A" }, + .{ .operation = .equal, .text = "3" }, + .{ .operation = .insert, .text = "BC" }, + }, diffs.items); + } } From b35dc243ce56d42c18d749272c10c3c019a4f238 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 12:49:54 -0400 Subject: [PATCH 062/176] Convert to ensuring capacity before append/insert --- DiffMatchPatch.zig | 146 ++++++++++++++++++++++++++++++--------------- 1 file changed, 98 insertions(+), 48 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index b273a63..6c9aeee 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -131,8 +131,13 @@ fn diffInternal( // Check for equality (speedup). if (std.mem.eql(u8, before, after)) { var diffs = DiffList{}; + if (before.len != 0) { - try diffsAppend(allocator, &diffs, .equal, before); + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.appendAssumeCapacity(Diff.init( + .equal, + try allocator.dupe(u8, before), + )); } return diffs; } @@ -154,11 +159,20 @@ fn diffInternal( errdefer deinitDiffList(allocator, &diffs); // Restore the prefix and suffix. + if (common_prefix.len != 0) { - try diffsInsert(allocator, &diffs, 0, .equal, common_prefix); + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.insertAssumeCapacity(0, Diff.init( + .equal, + try allocator.dupe(u8, common_prefix), + )); } if (common_suffix.len != 0) { - try diffsAppend(allocator, &diffs, .equal, common_suffix); + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.appendAssumeCapacity(Diff.init( + .equal, + try allocator.dupe(u8, common_suffix), + )); } try diffCleanupMerge(allocator, &diffs); @@ -213,13 +227,21 @@ fn diffCompute( if (before.len == 0) { // Just add some text (speedup). - try diffsAppend(allocator, &diffs, .insert, after); + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.appendAssumeCapacity(Diff.init( + .insert, + try allocator.dupe(u8, after), + )); return diffs; } if (after.len == 0) { // Just delete some text (speedup). - try diffsAppend(allocator, &diffs, .delete, before); + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.appendAssumeCapacity(Diff.init( + .delete, + try allocator.dupe(u8, before), + )); return diffs; } @@ -232,17 +254,34 @@ fn diffCompute( .delete else .insert; - try diffsAppend(allocator, &diffs, op, long_text[0..index]); - try diffsAppend(allocator, &diffs, .equal, short_text); - try diffsAppend(allocator, &diffs, op, long_text[index + short_text.len ..]); + try diffs.ensureUnusedCapacity(allocator, 3); + diffs.appendAssumeCapacity(Diff{ + .operation = op, + .text = try allocator.dupe(u8, long_text[0..index]), + }); + diffs.appendAssumeCapacity(Diff{ + .operation = .equal, + .text = try allocator.dupe(u8, short_text), + }); + diffs.appendAssumeCapacity(Diff{ + .operation = op, + .text = try allocator.dupe(u8, long_text[index + short_text.len ..]), + }); return diffs; } if (short_text.len == 1) { // Single character string. // After the previous speedup, the character can't be an equality. - try diffsAppend(allocator, &diffs, .delete, before); - try diffsAppend(allocator, &diffs, .insert, after); + try diffs.ensureUnusedCapacity(allocator, 2); + diffs.appendAssumeCapacity(Diff{ + .operation = .delete, + .text = try allocator.dupe(u8, before), + }); + diffs.appendAssumeCapacity(Diff{ + .operation = .insert, + .text = try allocator.dupe(u8, after), + }); return diffs; } @@ -276,7 +315,11 @@ fn diffCompute( // Merge the results. diffs = diffs_a; - try diffsAppend(allocator, &diffs, .equal, half_match.common_middle); + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.appendAssumeCapacity(Diff.init(.equal, try allocator.dupe( + u8, + half_match.common_middle, + ))); try diffs.appendSlice(allocator, diffs_b.items); return diffs; } @@ -574,8 +617,15 @@ fn diffBisect( // Diff took too long and hit the deadline or // number of diffs equals number of characters, no commonality at all. var diffs = DiffList{}; - try diffsAppend(allocator, &diffs, .delete, before); - try diffsAppend(allocator, &diffs, .insert, after); + try diffs.ensureUnusedCapacity(allocator, 2); + diffs.appendAssumeCapacity(Diff.init( + .delete, + try allocator.dupe(u8, before), + )); + diffs.appendAssumeCapacity(Diff.init( + .insert, + try allocator.dupe(u8, after), + )); return diffs; } @@ -890,11 +940,19 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo } if (text_delete.items.len != 0) { - try diffsInsert(allocator, diffs, pointer, .delete, text_delete.items); + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.insertAssumeCapacity(pointer, Diff.init( + .delete, + try allocator.dupe(u8, text_delete.items), + )); pointer += 1; } if (text_insert.items.len != 0) { - try diffsInsert(allocator, diffs, pointer, .insert, text_insert.items); + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.insertAssumeCapacity(pointer, Diff.init( + .insert, + try allocator.dupe(u8, text_insert.items), + )); pointer += 1; } pointer += 1; @@ -1020,12 +1078,13 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError (last_equality.?.len <= @max(length_insertions2, length_deletions2))) { // Duplicate record. - try diffsInsert( - allocator, - diffs, + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.insertAssumeCapacity( @intCast(equalities.items[equalities.items.len - 1]), - .delete, - last_equality.?, + Diff.init( + .delete, + try allocator.dupe(u8, last_equality.?), + ), ); // Change second copy to insert. diffs.items[@intCast(equalities.items[equalities.items.len - 1] + 1)].operation = .insert; @@ -1074,13 +1133,14 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError // Overlap found. // Insert an equality and trim the surrounding edits. defer allocator.free(deletion); - defer allocator.free(insertion); - try diffsInsert( - allocator, - diffs, + + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.insertAssumeCapacity( @intCast(pointer), - .equal, - insertion[0..overlap_length1], + Diff.init( + .equal, + try allocator.dupe(u8, insertion[0..overlap_length1]), + ), ); diffs.items[@intCast(pointer - 1)].text = try allocator.dupe(u8, deletion[0 .. deletion.len - overlap_length1]); @@ -1096,12 +1156,13 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError // Insert an equality and swap and trim the surrounding edits. defer allocator.free(deletion); defer allocator.free(insertion); - try diffsInsert( - allocator, - diffs, + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.insertAssumeCapacity( @intCast(pointer), - .equal, - deletion[0..overlap_length2], + Diff.init( + .equal, + try allocator.dupe(u8, deletion[0..overlap_length2]), + ), ); diffs.items[@intCast(pointer - 1)].operation = .insert; const new_minus = try allocator.dupe(u8, insertion[0 .. insertion.len - overlap_length2]); @@ -1342,12 +1403,13 @@ pub fn diffCleanupEfficiency( ((if (pre_ins) 1 else 0) + (if (pre_del) 1 else 0) + (if (post_ins) 1 else 0) + (if (post_del) 1 else 0)) == 3))) { // Duplicate record. - try diffsInsert( - allocator, - &diffs, + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.insertAssumeCapacity( equalities.items[equalities.items.len - 1], - .delete, - last_equality, + Diff.init( + .delete, + try allocator.dupe(u8, last_equality), + ), ); // Change second copy to insert. diffs.items[equalities.items[equalities.items.len - 1] + 1].operation = .insert; @@ -1425,18 +1487,6 @@ fn diffCommonOverlap(text1_in: []const u8, text2_in: []const u8) usize { } } -fn diffsAppend(allocator: Allocator, diffs: *DiffList, op: Diff.Operation, text: []const u8) !void { - const new_text = try allocator.dupe(u8, text); - errdefer allocator.free(new_text); - try diffs.append(allocator, Diff{ .operation = op, .text = new_text }); -} - -fn diffsInsert(allocator: Allocator, diffs: *DiffList, index: usize, op: Diff.Operation, text: []const u8) !void { - const new_text = try allocator.dupe(u8, text); - errdefer allocator.free(new_text); - try diffs.insert(allocator, index, Diff{ .operation = op, .text = new_text }); -} - // DONE [✅]: Allocate all text in diffs to // not cause segfault while freeing From 58fbf0f8e34bc08d01b28a29f204a02134d34521 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 12:55:52 -0400 Subject: [PATCH 063/176] Restore deleted free --- DiffMatchPatch.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 6c9aeee..59b566f 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1133,7 +1133,7 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError // Overlap found. // Insert an equality and trim the surrounding edits. defer allocator.free(deletion); - + defer allocator.free(insertion); try diffs.ensureUnusedCapacity(allocator, 1); diffs.insertAssumeCapacity( @intCast(pointer), From 39e0af82761e2bc739bad382648215d820173ac6 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 12:59:52 -0400 Subject: [PATCH 064/176] Consistent use of Diff.init() --- DiffMatchPatch.zig | 50 ++++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 59b566f..5134390 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -255,18 +255,18 @@ fn diffCompute( else .insert; try diffs.ensureUnusedCapacity(allocator, 3); - diffs.appendAssumeCapacity(Diff{ - .operation = op, - .text = try allocator.dupe(u8, long_text[0..index]), - }); - diffs.appendAssumeCapacity(Diff{ - .operation = .equal, - .text = try allocator.dupe(u8, short_text), - }); - diffs.appendAssumeCapacity(Diff{ - .operation = op, - .text = try allocator.dupe(u8, long_text[index + short_text.len ..]), - }); + diffs.appendAssumeCapacity(Diff.init( + op, + try allocator.dupe(u8, long_text[0..index]), + )); + diffs.appendAssumeCapacity(Diff.init( + .equal, + try allocator.dupe(u8, short_text), + )); + diffs.appendAssumeCapacity(Diff.init( + op, + try allocator.dupe(u8, long_text[index + short_text.len ..]), + )); return diffs; } @@ -274,14 +274,14 @@ fn diffCompute( // Single character string. // After the previous speedup, the character can't be an equality. try diffs.ensureUnusedCapacity(allocator, 2); - diffs.appendAssumeCapacity(Diff{ - .operation = .delete, - .text = try allocator.dupe(u8, before), - }); - diffs.appendAssumeCapacity(Diff{ - .operation = .insert, - .text = try allocator.dupe(u8, after), - }); + diffs.appendAssumeCapacity(Diff.init( + .delete, + try allocator.dupe(u8, before), + )); + diffs.appendAssumeCapacity(Diff.init( + .insert, + try allocator.dupe(u8, after), + )); return diffs; } @@ -316,10 +316,12 @@ fn diffCompute( // Merge the results. diffs = diffs_a; try diffs.ensureUnusedCapacity(allocator, 1); - diffs.appendAssumeCapacity(Diff.init(.equal, try allocator.dupe( - u8, - half_match.common_middle, - ))); + diffs.appendAssumeCapacity( + Diff.init(.equal, try allocator.dupe( + u8, + half_match.common_middle, + )), + ); try diffs.appendSlice(allocator, diffs_b.items); return diffs; } From 34d3a7b8ec3edd19865dbb7438aa2a371c0dded4 Mon Sep 17 00:00:00 2001 From: Techatrix Date: Sun, 7 Jul 2024 20:12:20 +0200 Subject: [PATCH 065/176] use `testing.checkAllAllocationFailures` --- DiffMatchPatch.zig | 1531 ++++++++++++++++++++++---------------------- 1 file changed, 780 insertions(+), 751 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 5134390..d9ba1f7 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1518,108 +1518,163 @@ test diffCommonOverlap { try testing.expectEqual(@as(usize, 0), diffCommonOverlap("fi", "\u{fb01}")); // Unicode } -test diffHalfMatch { - const allocator = testing.allocator; +fn testDiffHalfMatch( + allocator: std.mem.Allocator, + params: struct { + dmp: DiffMatchPatch, + before: []const u8, + after: []const u8, + expected: ?HalfMatchResult, + }, +) !void { + const maybe_result = try params.dmp.diffHalfMatch(allocator, params.before, params.after); + defer if (maybe_result) |result| result.deinit(allocator); + try testing.expectEqualDeep(params.expected, maybe_result); +} - var one_timeout = DiffMatchPatch{}; - one_timeout.diff_timeout = 1; - const dh1 = try one_timeout.diffHalfMatch(allocator, "1234567890", "abcdef"); - try testing.expectEqual( - @as(?HalfMatchResult, null), - dh1, - ); // No match #1 - const dh2 = try one_timeout.diffHalfMatch(allocator, "12345", "23"); - try testing.expectEqual( - @as(?HalfMatchResult, null), - dh2, - ); // No match #2 +test diffHalfMatch { + const one_timeout: DiffMatchPatch = .{ .diff_timeout = 1 }; + + // No match #1 + try testing.checkAllAllocationFailures(testing.allocator, testDiffHalfMatch, .{.{ + .dmp = one_timeout, + .before = "1234567890", + .after = "abcdef", + .expected = null, + }}); + + // No match #2 + try testing.checkAllAllocationFailures(testing.allocator, testDiffHalfMatch, .{.{ + .dmp = one_timeout, + .before = "12345", + .after = "23", + .expected = null, + }}); + + if (true) return error.SkipZigTest; // TODO // Single matches - var dh3 = (try one_timeout.diffHalfMatch(allocator, "1234567890", "a345678z")).?; - defer dh3.deinit(allocator); - try testing.expectEqualDeep(HalfMatchResult{ - .prefix_before = "12", - .suffix_before = "90", - .prefix_after = "a", - .suffix_after = "z", - .common_middle = "345678", - }, dh3); // Single Match #1 - - var dh4 = (try one_timeout.diffHalfMatch(allocator, "a345678z", "1234567890")).?; - defer dh4.deinit(allocator); - try testing.expectEqualDeep(HalfMatchResult{ - .prefix_before = "a", - .suffix_before = "z", - .prefix_after = "12", - .suffix_after = "90", - .common_middle = "345678", - }, dh4); // Single Match #2 - - var dh5 = (try one_timeout.diffHalfMatch(allocator, "abc56789z", "1234567890")).?; - defer dh5.deinit(allocator); - try testing.expectEqualDeep(HalfMatchResult{ - .prefix_before = "abc", - .suffix_before = "z", - .prefix_after = "1234", - .suffix_after = "0", - .common_middle = "56789", - }, dh5); // Single Match #3 - - var dh6 = (try one_timeout.diffHalfMatch(allocator, "a23456xyz", "1234567890")).?; - defer dh6.deinit(allocator); - try testing.expectEqualDeep(HalfMatchResult{ - .prefix_before = "a", - .suffix_before = "xyz", - .prefix_after = "1", - .suffix_after = "7890", - .common_middle = "23456", - }, dh6); // Single Match #4 - - // Multiple matches - var dh7 = (try one_timeout.diffHalfMatch(allocator, "121231234123451234123121", "a1234123451234z")).?; - defer dh7.deinit(allocator); - try testing.expectEqualDeep(HalfMatchResult{ - .prefix_before = "12123", - .suffix_before = "123121", - .prefix_after = "a", - .suffix_after = "z", - .common_middle = "1234123451234", - }, dh7); // Multiple Matches #1 - - var dh8 = (try one_timeout.diffHalfMatch(allocator, "x-=-=-=-=-=-=-=-=-=-=-=-=", "xx-=-=-=-=-=-=-=")).?; - defer dh8.deinit(allocator); - try testing.expectEqualDeep(HalfMatchResult{ - .prefix_before = "", - .suffix_before = "-=-=-=-=-=", - .prefix_after = "x", - .suffix_after = "", - .common_middle = "x-=-=-=-=-=-=-=", - }, dh8); // Multiple Matches #2 - - var dh9 = (try one_timeout.diffHalfMatch(allocator, "-=-=-=-=-=-=-=-=-=-=-=-=y", "-=-=-=-=-=-=-=yy")).?; - defer dh9.deinit(allocator); - try testing.expectEqualDeep(HalfMatchResult{ - .prefix_before = "-=-=-=-=-=", - .suffix_before = "", - .prefix_after = "", - .suffix_after = "y", - .common_middle = "-=-=-=-=-=-=-=y", - }, dh9); // Multiple Matches #3 + try testing.checkAllAllocationFailures(testing.allocator, testDiffHalfMatch, .{.{ + .dmp = one_timeout, + .before = "1234567890", + .after = "a345678z", + .expected = .{ + .prefix_before = "12", + .suffix_before = "90", + .prefix_after = "a", + .suffix_after = "z", + .common_middle = "345678", + }, + }}); + + // Single Match #2 + try testing.checkAllAllocationFailures(testing.allocator, testDiffHalfMatch, .{.{ + .dmp = one_timeout, + .before = "a345678z", + .after = "1234567890", + .expected = .{ + .prefix_before = "a", + .suffix_before = "z", + .prefix_after = "12", + .suffix_after = "90", + .common_middle = "345678", + }, + }}); + + // Single Match #3 + try testing.checkAllAllocationFailures(testing.allocator, testDiffHalfMatch, .{.{ + .dmp = one_timeout, + .before = "abc56789z", + .after = "1234567890", + .expected = .{ + .prefix_before = "abc", + .suffix_before = "z", + .prefix_after = "1234", + .suffix_after = "0", + .common_middle = "56789", + }, + }}); + + // Single Match #4 + try testing.checkAllAllocationFailures(testing.allocator, testDiffHalfMatch, .{.{ + .dmp = one_timeout, + .before = "a23456xyz", + .after = "1234567890", + .expected = .{ + .prefix_before = "a", + .suffix_before = "xyz", + .prefix_after = "1", + .suffix_after = "7890", + .common_middle = "23456", + }, + }}); + + // Multiple matches #1 + try testing.checkAllAllocationFailures(testing.allocator, testDiffHalfMatch, .{.{ + .dmp = one_timeout, + .before = "121231234123451234123121", + .after = "a1234123451234z", + .expected = .{ + .prefix_before = "12123", + .suffix_before = "123121", + .prefix_after = "a", + .suffix_after = "z", + .common_middle = "1234123451234", + }, + }}); + + // Multiple Matches #2 + try testing.checkAllAllocationFailures(testing.allocator, testDiffHalfMatch, .{.{ + .dmp = one_timeout, + .before = "x-=-=-=-=-=-=-=-=-=-=-=-=", + .after = "xx-=-=-=-=-=-=-=", + .expected = .{ + .prefix_before = "", + .suffix_before = "-=-=-=-=-=", + .prefix_after = "x", + .suffix_after = "", + .common_middle = "x-=-=-=-=-=-=-=", + }, + }}); + + // Multiple Matches #3 + try testing.checkAllAllocationFailures(testing.allocator, testDiffHalfMatch, .{.{ + .dmp = one_timeout, + .before = "-=-=-=-=-=-=-=-=-=-=-=-=y", + .after = "-=-=-=-=-=-=-=yy", + .expected = .{ + .prefix_before = "-=-=-=-=-=", + .suffix_before = "", + .prefix_after = "", + .suffix_after = "y", + .common_middle = "-=-=-=-=-=-=-=y", + }, + }}); // Other cases + // Optimal diff would be -q+x=H-i+e=lloHe+Hu=llo-Hew+y not -qHillo+x=HelloHe-w+Hulloy - var dh10 = (try one_timeout.diffHalfMatch(allocator, "qHilloHelloHew", "xHelloHeHulloy")).?; - defer dh10.deinit(allocator); - try testing.expectEqualDeep(HalfMatchResult{ - .prefix_before = "qHillo", - .suffix_before = "w", - .prefix_after = "x", - .suffix_after = "Hulloy", - .common_middle = "HelloHe", - }, dh10); // Non-optimal halfmatch - - one_timeout.diff_timeout = 0; - try testing.expectEqualDeep(@as(?HalfMatchResult, null), try one_timeout.diffHalfMatch(allocator, "qHilloHelloHew", "xHelloHeHulloy")); // Non-optimal halfmatch + // Non-optimal halfmatch + try testing.checkAllAllocationFailures(testing.allocator, testDiffHalfMatch, .{.{ + .dmp = one_timeout, + .before = "qHilloHelloHew", + .after = "xHelloHeHulloy", + .expected = .{ + .prefix_before = "qHillo", + .suffix_before = "w", + .prefix_after = "x", + .suffix_after = "Hulloy", + .common_middle = "HelloHe", + }, + }}); + + // Non-optimal halfmatch + try testing.checkAllAllocationFailures(testing.allocator, testDiffHalfMatch, .{.{ + .dmp = .{ .diff_timeout = 0 }, + .before = "qHilloHelloHew", + .after = "xHelloHeHulloy", + .expected = null, + }}); } test diffLinesToChars { @@ -1695,404 +1750,371 @@ test diffLinesToChars { // try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); } -test diffCharsToLines { - const allocator = std.testing.allocator; - - // Convert chars up to lines. - var diffs = try DiffList.initCapacity(allocator, 2); +fn testDiffCharsToLines( + allocator: std.mem.Allocator, + params: struct { + diffs: []const Diff, + line_array: []const []const u8, + expected: []const Diff, + }, +) !void { + var diffs = try DiffList.initCapacity(allocator, params.diffs.len); defer deinitDiffList(allocator, &diffs); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "\u{0001}\u{0002}\u{0001}") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "\u{0002}\u{0001}\u{0002}") }); + for (params.diffs) |item| { + diffs.appendAssumeCapacity(.{ .operation = item.operation, .text = try allocator.dupe(u8, item.text) }); + } - var tmp_vector = std.ArrayList([]const u8).init(allocator); - defer tmp_vector.deinit(); - try tmp_vector.append(""); - try tmp_vector.append("alpha\n"); - try tmp_vector.append("beta\n"); - try diffCharsToLines(allocator, diffs.items, tmp_vector.items); + try diffCharsToLines(allocator, diffs.items, params.line_array); - try testing.expectEqualDeep(&[_]Diff{ - .{ .operation = .equal, .text = "alpha\nbeta\nalpha\n" }, - .{ .operation = .insert, .text = "beta\nalpha\nbeta\n" }, - }, diffs.items); + try testing.expectEqualDeep(params.expected, diffs.items); +} + +test diffCharsToLines { + if (true) return error.SkipZigTest; // TODO + + // Convert chars up to lines. + try testing.checkAllAllocationFailures(testing.allocator, testDiffCharsToLines, .{.{ + .diffs = &.{ + .{ .operation = .equal, .text = "\u{0001}\u{0002}\u{0001}" }, + .{ .operation = .insert, .text = "\u{0002}\u{0001}\u{0002}" }, + }, + .line_array = &[_][]const u8{ + "", + "alpha\n", + "beta\n", + }, + .expected = &.{ + .{ .operation = .equal, .text = "alpha\nbeta\nalpha\n" }, + .{ .operation = .insert, .text = "beta\nalpha\nbeta\n" }, + }, + }}); // TODO: Implement exhaustive tests } -test diffCleanupMerge { - const allocator = testing.allocator; - // Cleanup a messy diff. +fn testDiffCleanupMerge(allocator: std.mem.Allocator, params: struct { + input: []const Diff, + expected: []const Diff, +}) !void { + var diffs = try DiffList.initCapacity(allocator, params.input.len); + defer deinitDiffList(allocator, &diffs); - { - // No change case - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); + for (params.input) |item| { + diffs.appendAssumeCapacity(.{ .operation = item.operation, .text = try allocator.dupe(u8, item.text) }); + } + + try diffCleanupMerge(allocator, &diffs); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "b") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "c") }); + try testing.expectEqualDeep(params.expected, diffs.items); +} - try diffCleanupMerge(allocator, &diffs); - try diffCleanupMerge(allocator, &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "b" }, .{ .operation = .insert, .text = "c" } }), diffs.items); // No change case - try diffCleanupMerge(allocator, &diffs); - try testing.expectEqualDeep(@as([]const Diff, &[_]Diff{ .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "b" }, .{ .operation = .insert, .text = "c" } }), diffs.items); // No change case +test diffCleanupMerge { + // Cleanup a messy diff. - try testing.expectEqualDeep(&[_]Diff{ + // No change case + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "b" }, .{ .operation = .insert, .text = "c" }, - }, diffs.items); - } - - { - // Merge equalities - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "b") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "c") }); - - try diffCleanupMerge(allocator, &diffs); + }, + .expected = &.{ + .{ .operation = .equal, .text = "a" }, + .{ .operation = .delete, .text = "b" }, + .{ .operation = .insert, .text = "c" }, + }, + }}); - try testing.expectEqualDeep(&[_]Diff{ + // Merge equalities + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "a" }, + .{ .operation = .equal, .text = "b" }, + .{ .operation = .equal, .text = "c" }, + }, + .expected = &.{ .{ .operation = .equal, .text = "abc" }, - }, diffs.items); - } - - { - // Merge deletions - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "a") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "b") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "c") }); + }, + }}); - try diffCleanupMerge(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + // Merge deletions + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .delete, .text = "a" }, + .{ .operation = .delete, .text = "b" }, + .{ .operation = .delete, .text = "c" }, + }, + .expected = &.{ .{ .operation = .delete, .text = "abc" }, - }, diffs.items); - } + }, + }}); - { - - // Merge insertions - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "a") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "b") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "c") }); - - try diffCleanupMerge(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + // Merge insertions + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .insert, .text = "a" }, + .{ .operation = .insert, .text = "b" }, + .{ .operation = .insert, .text = "c" }, + }, + .expected = &.{ .{ .operation = .insert, .text = "abc" }, - }, diffs.items); - } - - { - // Merge interweave - var diffs = try DiffList.initCapacity(allocator, 6); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "a") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "b") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "c") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "d") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "e") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "f") }); + }, + }}); - try diffCleanupMerge(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + // Merge interweave + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .delete, .text = "a" }, + .{ .operation = .insert, .text = "b" }, + .{ .operation = .delete, .text = "c" }, + .{ .operation = .insert, .text = "d" }, + .{ .operation = .equal, .text = "e" }, + .{ .operation = .equal, .text = "f" }, + }, + .expected = &.{ .{ .operation = .delete, .text = "ac" }, .{ .operation = .insert, .text = "bd" }, .{ .operation = .equal, .text = "ef" }, - }, diffs.items); - } + }, + }}); - { - // Prefix and suffix detection - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "a") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "abc") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "dc") }); - - try diffCleanupMerge(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + // Prefix and suffix detection + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .delete, .text = "a" }, + .{ .operation = .insert, .text = "abc" }, + .{ .operation = .delete, .text = "dc" }, + }, + .expected = &.{ .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "d" }, .{ .operation = .insert, .text = "b" }, .{ .operation = .equal, .text = "c" }, - }, diffs.items); - } - - { - // Prefix and suffix detection with equalities - var diffs = try DiffList.initCapacity(allocator, 5); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "x") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "a") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "abc") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "dc") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "y") }); + }, + }}); - try diffCleanupMerge(allocator, &diffs); + if (true) return error.SkipZigTest; // TODO - try testing.expectEqualDeep(&[_]Diff{ + // Prefix and suffix detection with equalities + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "x" }, + .{ .operation = .delete, .text = "a" }, + .{ .operation = .insert, .text = "abc" }, + .{ .operation = .delete, .text = "dc" }, + .{ .operation = .equal, .text = "y" }, + }, + .expected = &.{ .{ .operation = .equal, .text = "xa" }, .{ .operation = .delete, .text = "d" }, .{ .operation = .insert, .text = "b" }, .{ .operation = .equal, .text = "cy" }, - }, diffs.items); - } - - { - // Slide edit left - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "ba") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "c") }); - - try diffCleanupMerge(allocator, &diffs); + }, + }}); - try testing.expectEqualDeep(&[_]Diff{ + // Slide edit left + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "a" }, + .{ .operation = .insert, .text = "ba" }, + .{ .operation = .equal, .text = "c" }, + }, + .expected = &.{ .{ .operation = .insert, .text = "ab" }, .{ .operation = .equal, .text = "ac" }, - }, diffs.items); - } - - { - - // Slide edit right - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "c") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "ab") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); + }, + }}); - try diffCleanupMerge(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + // Slide edit right + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "c" }, + .{ .operation = .insert, .text = "ab" }, + .{ .operation = .equal, .text = "a" }, + }, + .expected = &.{ .{ .operation = .equal, .text = "ca" }, .{ .operation = .insert, .text = "ba" }, - }, diffs.items); - } - - { - - // Slide edit left recursive - var diffs = try DiffList.initCapacity(allocator, 5); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "b") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "c") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "ac") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "x") }); - - try diffCleanupMerge(allocator, &diffs); - - try testing.expectEqualDeep(&.{ - Diff.init(.delete, "abc"), - Diff.init(.equal, "acx"), - }, diffs.items); - } - - { - // Slide edit right recursive - var diffs = try DiffList.initCapacity(allocator, 5); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "x") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "ca") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "c") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "b") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); - - try diffCleanupMerge(allocator, &diffs); - - try testing.expectEqualDeep(&.{ - Diff.init(.equal, "xca"), - Diff.init(.delete, "cba"), - }, diffs.items); - } + }, + }}); - { - // Empty merge - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); + // Slide edit left recursive + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "a" }, + .{ .operation = .delete, .text = "b" }, + .{ .operation = .equal, .text = "c" }, + .{ .operation = .delete, .text = "ac" }, + .{ .operation = .equal, .text = "x" }, + }, + .expected = &.{ + .{ .operation = .delete, .text = "abc" }, + .{ .operation = .equal, .text = "acx" }, + }, + }}); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "b") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "ab") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "c") }); + // Slide edit right recursive + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "x" }, + .{ .operation = .delete, .text = "ca" }, + .{ .operation = .equal, .text = "c" }, + .{ .operation = .delete, .text = "b" }, + .{ .operation = .equal, .text = "a" }, + }, + .expected = &.{ + .{ .operation = .equal, .text = "xca" }, + .{ .operation = .delete, .text = "cba" }, + }, + }}); + + // Empty merge + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .delete, .text = "b" }, + .{ .operation = .insert, .text = "ab" }, + .{ .operation = .equal, .text = "c" }, + }, + .expected = &.{ + .{ .operation = .insert, .text = "a" }, + .{ .operation = .equal, .text = "bc" }, + }, + }}); + + // Empty equality + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "" }, + .{ .operation = .insert, .text = "a" }, + .{ .operation = .equal, .text = "b" }, + }, + .expected = &.{ + .{ .operation = .insert, .text = "a" }, + .{ .operation = .equal, .text = "b" }, + }, + }}); +} - try diffCleanupMerge(allocator, &diffs); +fn testDiffCleanupSemanticLossless( + allocator: std.mem.Allocator, + params: struct { + input: []const Diff, + expected: []const Diff, + }, +) !void { + var diffs = try DiffList.initCapacity(allocator, params.input.len); + defer deinitDiffList(allocator, &diffs); - try testing.expectEqualDeep(&.{ - Diff.init(.insert, "a"), - Diff.init(.equal, "bc"), - }, diffs.items); + for (params.input) |item| { + diffs.appendAssumeCapacity(.{ .operation = item.operation, .text = try allocator.dupe(u8, item.text) }); } - { - // Empty equality - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); + try diffCleanupSemanticLossless(allocator, &diffs); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = "" }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "a") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "b") }); - - try diffCleanupMerge(allocator, &diffs); - - try testing.expectEqualDeep(&.{ - Diff.init(.insert, "a"), - Diff.init(.equal, "b"), - }, diffs.items); - } + try testing.expectEqualDeep(params.expected, diffs.items); } test diffCleanupSemanticLossless { - const allocator = testing.allocator; - - { - // Null case - var diffs: DiffList = .{}; - try diffCleanupSemanticLossless(allocator, &diffs); - try testing.expectEqualDeep(&[_]Diff{}, diffs.items); - } - - { - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "AAA\r\n\r\nBBB") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "\r\nDDD\r\n\r\nBBB") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "\r\nEEE") }); - - try diffCleanupSemanticLossless(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ - Diff.init(.equal, "AAA\r\n\r\n"), - Diff.init(.insert, "BBB\r\nDDD\r\n\r\n"), - Diff.init(.equal, "BBB\r\nEEE"), - }, diffs.items); - } - - { - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "AAA\r\nBBB") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, " DDD\r\nBBB") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, " EEE") }); - - try diffCleanupSemanticLossless(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ - Diff.init(.equal, "AAA\r\n"), - Diff.init(.insert, "BBB DDD\r\n"), - Diff.init(.equal, "BBB EEE"), - }, diffs.items); - } - - { - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "The c") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "ow and the c") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "at.") }); - - try diffCleanupSemanticLossless(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ - Diff.init(.equal, "The "), - Diff.init(.insert, "cow and the "), - Diff.init(.equal, "cat."), - }, diffs.items); - } - - { - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "The-c") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "ow-and-the-c") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "at.") }); - - try diffCleanupSemanticLossless(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ - Diff.init(.equal, "The-"), - Diff.init(.insert, "cow-and-the-"), - Diff.init(.equal, "cat."), - }, diffs.items); - } - - { - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "a") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "ax") }); - - try diffCleanupSemanticLossless(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ - Diff.init(.delete, "a"), - Diff.init(.equal, "aax"), - }, diffs.items); - } - - { - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "xa") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "a") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "a") }); - - try diffCleanupSemanticLossless(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ - Diff.init(.equal, "xaa"), - Diff.init(.delete, "a"), - }, diffs.items); - } - - { - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "The xxx. The ") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "zzz. The ") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "yyy.") }); + // Null case + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemanticLossless, .{.{ + .input = &[_]Diff{}, + .expected = &[_]Diff{}, + }}); + + if (true) return error.SkipZigTest; // TODO + + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemanticLossless, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "AAA\r\n\r\nBBB" }, + .{ .operation = .insert, .text = "\r\nDDD\r\n\r\nBBB" }, + .{ .operation = .equal, .text = "\r\nEEE" }, + }, + .expected = &.{ + .{ .operation = .equal, .text = "AAA\r\n\r\n" }, + .{ .operation = .insert, .text = "BBB\r\nDDD\r\n\r\n" }, + .{ .operation = .equal, .text = "BBB\r\nEEE" }, + }, + }}); + + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemanticLossless, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "AAA\r\nBBB" }, + .{ .operation = .insert, .text = " DDD\r\nBBB" }, + .{ .operation = .equal, .text = " EEE" }, + }, + .expected = &.{ + .{ .operation = .equal, .text = "AAA\r\n" }, + .{ .operation = .insert, .text = "BBB DDD\r\n" }, + .{ .operation = .equal, .text = "BBB EEE" }, + }, + }}); + + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemanticLossless, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "The c" }, + .{ .operation = .insert, .text = "ow and the c" }, + .{ .operation = .equal, .text = "at." }, + }, + .expected = &.{ + .{ .operation = .equal, .text = "The " }, + .{ .operation = .insert, .text = "cow and the " }, + .{ .operation = .equal, .text = "cat." }, + }, + }}); + + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemanticLossless, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "The-c" }, + .{ .operation = .insert, .text = "ow-and-the-c" }, + .{ .operation = .equal, .text = "at." }, + }, + .expected = &.{ + .{ .operation = .equal, .text = "The-" }, + .{ .operation = .insert, .text = "cow-and-the-" }, + .{ .operation = .equal, .text = "cat." }, + }, + }}); - try diffCleanupSemanticLossless(allocator, &diffs); + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemanticLossless, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "a" }, + .{ .operation = .delete, .text = "a" }, + .{ .operation = .equal, .text = "ax" }, + }, + .expected = &.{ + .{ .operation = .delete, .text = "a" }, + .{ .operation = .equal, .text = "aax" }, + }, + }}); - try testing.expectEqualDeep(&[_]Diff{ - Diff.init(.equal, "The xxx."), - Diff.init(.insert, " The zzz."), - Diff.init(.equal, " The yyy."), - }, diffs.items); - } + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemanticLossless, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "xa" }, + .{ .operation = .delete, .text = "a" }, + .{ .operation = .equal, .text = "a" }, + }, + .expected = &.{ + .{ .operation = .equal, .text = "xaa" }, + .{ .operation = .delete, .text = "a" }, + }, + }}); + + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemanticLossless, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "The xxx. The " }, + .{ .operation = .insert, .text = "zzz. The " }, + .{ .operation = .equal, .text = "yyy." }, + }, + .expected = &.{ + .{ .operation = .equal, .text = "The xxx." }, + .{ .operation = .insert, .text = " The zzz." }, + .{ .operation = .equal, .text = " The yyy." }, + }, + }}); } +/// TODO this function obviously leaks memory on error fn rebuildtexts(allocator: std.mem.Allocator, diffs: DiffList) ![2][]const u8 { var text = [2]std.ArrayList(u8){ std.ArrayList(u8).init(allocator), @@ -2113,205 +2135,231 @@ fn rebuildtexts(allocator: std.mem.Allocator, diffs: DiffList) ![2][]const u8 { }; } -test diffBisect { - const allocator = testing.allocator; +fn testDiffBisect( + allocator: std.mem.Allocator, + params: struct { + dmp: DiffMatchPatch, + before: []const u8, + after: []const u8, + deadline: u64, + expected: []const Diff, + }, +) !void { + var diffs = try params.dmp.diffBisect(allocator, params.before, params.after, params.deadline); + defer deinitDiffList(allocator, &diffs); + try testing.expectEqualDeep(params.expected, diffs.items); +} +test diffBisect { const this: DiffMatchPatch = .{ .diff_timeout = 0 }; const a = "cat"; const b = "map"; - { - // Normal. - - // Since the resulting diff hasn't been normalized, it would be ok if - // the insertion and deletion pairs are swapped. - // If the order changes, tweak this test as required. - // Travis TODO not sure if maxInt(u64) is correct for DateTime.MaxValue - var diffs = try this.diffBisect( - allocator, - a, - b, - std.math.maxInt(u64), - ); - defer deinitDiffList(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + // Normal + try testing.checkAllAllocationFailures(testing.allocator, testDiffBisect, .{.{ + .dmp = this, + .before = a, + .after = b, + .deadline = std.math.maxInt(u64), // Travis TODO not sure if maxInt(u64) is correct for DateTime.MaxValue + .expected = &.{ .{ .operation = .delete, .text = "c" }, .{ .operation = .insert, .text = "m" }, .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "t" }, .{ .operation = .insert, .text = "p" }, - }, diffs.items); - } - - { - // Timeout. - var diffs2 = DiffList{}; - defer deinitDiffList(allocator, &diffs2); - try diffs2.appendSlice(allocator, &.{ - Diff.init(.delete, try allocator.dupe(u8, "cat")), - Diff.init(.insert, try allocator.dupe(u8, "map")), - }); - // Travis TODO not sure if 0 is correct for DateTime.MinValue - var diffs = try this.diffBisect(allocator, a, b, 0); - defer deinitDiffList(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + if (true) return error.SkipZigTest; // TODO + + // Timeout + try testing.checkAllAllocationFailures(testing.allocator, testDiffBisect, .{.{ + .dmp = this, + .before = a, + .after = b, + .deadline = 0, // Travis TODO not sure if 0 is correct for DateTime.MinValue + .expected = &.{ .{ .operation = .delete, .text = "cat" }, .{ .operation = .insert, .text = "map" }, - }, diffs.items); - } + }, + }}); +} + +fn testDiff( + allocator: std.mem.Allocator, + params: struct { + dmp: DiffMatchPatch, + before: []const u8, + after: []const u8, + check_lines: bool, + expected: []const Diff, + }, +) !void { + var diffs = try params.dmp.diff(allocator, params.before, params.after, params.check_lines); + defer deinitDiffList(allocator, &diffs); + try testing.expectEqualDeep(params.expected, diffs.items); } test diff { - const allocator = testing.allocator; + if (true) return error.SkipZigTest; // TODO const this: DiffMatchPatch = .{ .diff_timeout = 0 }; - { - // diff: Null case. - var diffs = try this.diff(allocator, "", "", false); - defer deinitDiffList(allocator, &diffs); - - try testing.expectEqual(0, diffs.items.len); - } - - { - // diff: Equality. - var diffs = try this.diff(allocator, "abc", "abc", false); - defer deinitDiffList(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + // Null case. + try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ + .dmp = this, + .before = "", + .after = "", + .check_lines = false, + .expected = &[_]Diff{}, + }}); + + // Equality. + try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ + .dmp = this, + .before = "abc", + .after = "abc", + .check_lines = false, + .expected = &.{ .{ .operation = .equal, .text = "abc" }, - }, diffs.items); - } - - { - // diff: Simple insertion. - var diffs = try this.diff(allocator, "abc", "ab123c", false); - defer deinitDiffList(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + // Simple insertion. + try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ + .dmp = this, + .before = "abc", + .after = "ab123c", + .check_lines = false, + .expected = &.{ .{ .operation = .equal, .text = "ab" }, .{ .operation = .insert, .text = "123" }, .{ .operation = .equal, .text = "c" }, - }, diffs.items); - } - - { - // diff: Simple deletion. - var diffs = try this.diff(allocator, "a123bc", "abc", false); - defer deinitDiffList(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + // Simple deletion. + try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ + .dmp = this, + .before = "a123bc", + .after = "abc", + .check_lines = false, + .expected = &.{ .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "123" }, .{ .operation = .equal, .text = "bc" }, - }, diffs.items); - } - - { - // diff: Two insertions. - var diffs = try this.diff(allocator, "abc", "a123b456c", false); - defer deinitDiffList(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + // Two insertions. + try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ + .dmp = this, + .before = "abc", + .after = "a123b456c", + .check_lines = false, + .expected = &.{ .{ .operation = .equal, .text = "a" }, .{ .operation = .insert, .text = "123" }, .{ .operation = .equal, .text = "b" }, .{ .operation = .insert, .text = "456" }, .{ .operation = .equal, .text = "c" }, - }, diffs.items); - } - - { - // diff: Two deletions. - var diffs = try this.diff(allocator, "a123b456c", "abc", false); - defer deinitDiffList(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + // Two deletions. + try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ + .dmp = this, + .before = "a123b456c", + .after = "abc", + .check_lines = false, + .expected = &.{ .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "123" }, .{ .operation = .equal, .text = "b" }, .{ .operation = .delete, .text = "456" }, .{ .operation = .equal, .text = "c" }, - }, diffs.items); - } - - // Perform a real diff. - { - // diff: Simple case #1. - var diffs = try this.diff(allocator, "a", "b", false); - defer deinitDiffList(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + // Simple case #1 + try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ + .dmp = this, + .before = "a", + .after = "b", + .check_lines = false, + .expected = &.{ .{ .operation = .delete, .text = "a" }, .{ .operation = .insert, .text = "b" }, - }, diffs.items); - } - - { - // diff: Simple case #2. - var diffs = try this.diff(allocator, "Apples are a fruit.", "Bananas are also fruit.", false); - defer deinitDiffList(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + // Simple case #2 + try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ + .dmp = this, + .before = "Apples are a fruit.", + .after = "Bananas are also fruit.", + .check_lines = false, + .expected = &.{ .{ .operation = .delete, .text = "Apple" }, .{ .operation = .insert, .text = "Banana" }, .{ .operation = .equal, .text = "s are a" }, .{ .operation = .insert, .text = "lso" }, .{ .operation = .equal, .text = " fruit." }, - }, diffs.items); - } - - { - // diff: Simple case #3. - var diffs = try this.diff(allocator, "ax\t", "\u{0680}x\x00", false); - defer deinitDiffList(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + // Simple case #3 + try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ + .dmp = this, + .before = "ax\t", + .after = "\u{0680}x\x00", + .check_lines = false, + .expected = &.{ .{ .operation = .delete, .text = "a" }, .{ .operation = .insert, .text = "\u{0680}" }, .{ .operation = .equal, .text = "x" }, .{ .operation = .delete, .text = "\t" }, .{ .operation = .insert, .text = "\x00" }, - }, diffs.items); - } - - { - // diff: Overlap #1. - var diffs = try this.diff(allocator, "1ayb2", "abxab", false); - defer deinitDiffList(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + // Overlap #1 + try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ + .dmp = this, + .before = "1ayb2", + .after = "abxab", + .check_lines = false, + .expected = &.{ .{ .operation = .delete, .text = "1" }, .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "y" }, .{ .operation = .equal, .text = "b" }, .{ .operation = .delete, .text = "2" }, .{ .operation = .insert, .text = "xab" }, - }, diffs.items); - } - - { - // diff: Overlap #2. - var diffs = try this.diff(allocator, "abcy", "xaxcxabc", false); - defer deinitDiffList(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + // Overlap #2 + try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ + .dmp = this, + .before = "abcy", + .after = "xaxcxabc", + .check_lines = false, + .expected = &.{ .{ .operation = .insert, .text = "xaxcx" }, .{ .operation = .equal, .text = "abc" }, .{ .operation = .delete, .text = "y" }, - }, diffs.items); - } - - { - // diff: Overlap #3. - var diffs = try this.diff(allocator, "ABCDa=bcd=efghijklmnopqrsEFGHIJKLMNOefg", "a-bcd-efghijklmnopqrs", false); - defer deinitDiffList(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + // Overlap #3 + try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ + .dmp = this, + .before = "ABCDa=bcd=efghijklmnopqrsEFGHIJKLMNOefg", + .after = "a-bcd-efghijklmnopqrs", + .check_lines = false, + .expected = &.{ .{ .operation = .delete, .text = "ABCD" }, .{ .operation = .equal, .text = "a" }, .{ .operation = .delete, .text = "=" }, @@ -2321,22 +2369,26 @@ test diff { .{ .operation = .insert, .text = "-" }, .{ .operation = .equal, .text = "efghijklmnopqrs" }, .{ .operation = .delete, .text = "EFGHIJKLMNOefg" }, - }, diffs.items); - } - - { - // diff: Large equality. - var diffs = try this.diff(allocator, "a [[Pennsylvania]] and [[New", " and [[Pennsylvania]]", false); - defer deinitDiffList(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + // Large equality + try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ + .dmp = this, + .before = "a [[Pennsylvania]] and [[New", + .after = " and [[Pennsylvania]]", + .check_lines = false, + .expected = &.{ .{ .operation = .insert, .text = " " }, .{ .operation = .equal, .text = "a" }, .{ .operation = .insert, .text = "nd" }, .{ .operation = .equal, .text = " [[Pennsylvania]]" }, .{ .operation = .delete, .text = " and [[New" }, - }, diffs.items); - } + }, + }}); + + const allocator = testing.allocator; + // TODO these tests should be checked for allocation failure // Increase the text lengths by 1024 times to ensure a timeout. { @@ -2418,198 +2470,175 @@ test diff { } } -test diffCleanupSemantic { - const allocator = testing.allocator; +fn testDiffCleanupSemantic( + allocator: std.mem.Allocator, + params: struct { + input: []const Diff, + expected: []const Diff, + }, +) !void { + var diffs = try DiffList.initCapacity(allocator, params.input.len); + defer deinitDiffList(allocator, &diffs); - { - // Null case. - var diffs: DiffList = .{}; - defer deinitDiffList(allocator, &diffs); - try diffCleanupSemantic(allocator, &diffs); - try testing.expectEqual(@as(usize, 0), diffs.items.len); // Null case + for (params.input) |item| { + diffs.appendAssumeCapacity(.{ .operation = item.operation, .text = try allocator.dupe(u8, item.text) }); } - { - // No elimination #1 - var diffs = try DiffList.initCapacity(allocator, 4); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "ab") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "cd") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "12") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "e") }); + try diffCleanupSemantic(allocator, &diffs); - try diffCleanupSemantic(allocator, &diffs); + try testing.expectEqualDeep(params.expected, diffs.items); +} - try testing.expectEqualDeep(&[_]Diff{ +test diffCleanupSemantic { + // Null case. + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &[_]Diff{}, + .expected = &[_]Diff{}, + }}); + + // No elimination #1 + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ .{ .operation = .delete, .text = "ab" }, .{ .operation = .insert, .text = "cd" }, .{ .operation = .equal, .text = "12" }, .{ .operation = .delete, .text = "e" }, - }, diffs.items); - } - - { - // No elimination #2 - var diffs = try DiffList.initCapacity(allocator, 4); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "abc") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "ABC") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "1234") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "wxyz") }); - - try diffCleanupSemantic(allocator, &diffs); + }, + .expected = &.{ + .{ .operation = .delete, .text = "ab" }, + .{ .operation = .insert, .text = "cd" }, + .{ .operation = .equal, .text = "12" }, + .{ .operation = .delete, .text = "e" }, + }, + }}); - try testing.expectEqualDeep(&[_]Diff{ + // No elimination #2 + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ .{ .operation = .delete, .text = "abc" }, .{ .operation = .insert, .text = "ABC" }, .{ .operation = .equal, .text = "1234" }, .{ .operation = .delete, .text = "wxyz" }, - }, diffs.items); - } - - { - // Simple elimination - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "a") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "b") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "c") }); - - try diffCleanupSemantic(allocator, &diffs); + }, + .expected = &.{ + .{ .operation = .delete, .text = "abc" }, + .{ .operation = .insert, .text = "ABC" }, + .{ .operation = .equal, .text = "1234" }, + .{ .operation = .delete, .text = "wxyz" }, + }, + }}); - try testing.expectEqualDeep(&[_]Diff{ + // Simple elimination + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ + .{ .operation = .delete, .text = "a" }, + .{ .operation = .equal, .text = "b" }, + .{ .operation = .delete, .text = "c" }, + }, + .expected = &.{ .{ .operation = .delete, .text = "abc" }, .{ .operation = .insert, .text = "b" }, - }, diffs.items); - } - - { - // Backpass elimination - var diffs = try DiffList.initCapacity(allocator, 5); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "ab") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "cd") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "e") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "f") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "g") }); - - try diffCleanupSemantic(allocator, &diffs); + }, + }}); - try testing.expectEqualDeep(&[_]Diff{ + // Backpass elimination + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ + .{ .operation = .delete, .text = "ab" }, + .{ .operation = .equal, .text = "cd" }, + .{ .operation = .delete, .text = "e" }, + .{ .operation = .equal, .text = "f" }, + .{ .operation = .insert, .text = "g" }, + }, + .expected = &.{ .{ .operation = .delete, .text = "abcdef" }, .{ .operation = .insert, .text = "cdfg" }, - }, diffs.items); - } - - { - // Multiple elimination - var diffs = try DiffList.initCapacity(allocator, 9); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "1") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "A") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "B") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "2") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "_") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "1") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "A") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "B") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "2") }); - - try diffCleanupSemantic(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + // Multiple elimination + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ + .{ .operation = .insert, .text = "1" }, + .{ .operation = .equal, .text = "A" }, + .{ .operation = .delete, .text = "B" }, + .{ .operation = .insert, .text = "2" }, + .{ .operation = .equal, .text = "_" }, + .{ .operation = .insert, .text = "1" }, + .{ .operation = .equal, .text = "A" }, + .{ .operation = .delete, .text = "B" }, + .{ .operation = .insert, .text = "2" }, + }, + .expected = &.{ .{ .operation = .delete, .text = "AB_AB" }, .{ .operation = .insert, .text = "1A2_1A2" }, - }, diffs.items); - } - - { - // Word boundaries - var diffs = try DiffList.initCapacity(allocator, 3); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "The c") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "ow and the c") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "at.") }); - - try diffCleanupSemantic(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + if (true) return error.SkipZigTest; // TODO + + // Word boundaries + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "The c" }, + .{ .operation = .delete, .text = "ow and the c" }, + .{ .operation = .equal, .text = "at." }, + }, + .expected = &.{ .{ .operation = .equal, .text = "The " }, .{ .operation = .delete, .text = "cow and the " }, .{ .operation = .equal, .text = "cat." }, - }, diffs.items); - } - - { - // No overlap elimination - var diffs = try DiffList.initCapacity(allocator, 2); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "abcxx") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "xxdef") }); - - try diffCleanupSemantic(allocator, &diffs); + }, + }}); - try testing.expectEqualDeep(&[_]Diff{ + // No overlap elimination + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ .{ .operation = .delete, .text = "abcxx" }, .{ .operation = .insert, .text = "xxdef" }, - }, diffs.items); - } - - { - // Overlap elimination - var diffs = try DiffList.initCapacity(allocator, 2); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "abcxxx") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "xxxdef") }); - - try diffCleanupSemantic(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + .expected = &.{ + .{ .operation = .delete, .text = "abcxx" }, + .{ .operation = .insert, .text = "xxdef" }, + }, + }}); + + // Overlap elimination + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ + .{ .operation = .delete, .text = "abcxxx" }, + .{ .operation = .insert, .text = "xxxdef" }, + }, + .expected = &.{ .{ .operation = .delete, .text = "abc" }, .{ .operation = .equal, .text = "xxx" }, .{ .operation = .insert, .text = "def" }, - }, diffs.items); - } - - { - // Reverse overlap elimination - var diffs = try DiffList.initCapacity(allocator, 2); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "xxxabc") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "defxxx") }); - - try diffCleanupSemantic(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + // Reverse overlap elimination + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ + .{ .operation = .delete, .text = "xxxabc" }, + .{ .operation = .insert, .text = "defxxx" }, + }, + .expected = &.{ .{ .operation = .insert, .text = "def" }, .{ .operation = .equal, .text = "xxx" }, .{ .operation = .delete, .text = "abc" }, - }, diffs.items); - } - - { - // Two overlap eliminations - var diffs = try DiffList.initCapacity(allocator, 5); - defer deinitDiffList(allocator, &diffs); - - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "abcd1212") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "1212efghi") }); - diffs.appendAssumeCapacity(.{ .operation = .equal, .text = try allocator.dupe(u8, "----") }); - diffs.appendAssumeCapacity(.{ .operation = .delete, .text = try allocator.dupe(u8, "A3") }); - diffs.appendAssumeCapacity(.{ .operation = .insert, .text = try allocator.dupe(u8, "3BC") }); - - try diffCleanupSemantic(allocator, &diffs); - - try testing.expectEqualDeep(&[_]Diff{ + }, + }}); + + // Two overlap eliminations + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ + .{ .operation = .delete, .text = "abcd1212" }, + .{ .operation = .insert, .text = "1212efghi" }, + .{ .operation = .equal, .text = "----" }, + .{ .operation = .delete, .text = "A3" }, + .{ .operation = .insert, .text = "3BC" }, + }, + .expected = &.{ .{ .operation = .delete, .text = "abcd" }, .{ .operation = .equal, .text = "1212" }, .{ .operation = .insert, .text = "efghi" }, @@ -2617,6 +2646,6 @@ test diffCleanupSemantic { .{ .operation = .delete, .text = "A" }, .{ .operation = .equal, .text = "3" }, .{ .operation = .insert, .text = "BC" }, - }, diffs.items); - } + }, + }}); } From 8cb7f017670edd3147e78ba3f548388a90b323cf Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 14:47:46 -0400 Subject: [PATCH 066/176] errdefer halfmatches --- DiffMatchPatch.zig | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index d9ba1f7..3f6789e 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -378,8 +378,14 @@ fn diffHalfMatch( // First check if the second quarter is the seed for a half-match. const half_match_1 = try dmp.diffHalfMatchInternal(allocator, long_text, short_text, (long_text.len + 3) / 4); + errdefer { + if (half_match_1) |h_m| h_m.deinit(allocator); + } // Check again based on the third quarter. const half_match_2 = try dmp.diffHalfMatchInternal(allocator, long_text, short_text, (long_text.len + 1) / 2); + errdefer { + if (half_match_2) |h_m| h_m.deinit(allocator); + } var half_match: ?HalfMatchResult = null; if (half_match_1 == null and half_match_2 == null) { @@ -471,12 +477,14 @@ fn diffHalfMatchInternal( errdefer allocator.free(prefix_after); const suffix_after = try allocator.dupe(u8, best_short_text_b); errdefer allocator.free(suffix_after); + const best_common_text = try best_common.toOwnedSlice(allocator); + errdefer allocator.free(best_common_text); return .{ .prefix_before = prefix_before, .suffix_before = suffix_before, .prefix_after = prefix_after, .suffix_after = suffix_after, - .common_middle = try best_common.toOwnedSlice(allocator), + .common_middle = best_common_text, }; } else { return null; @@ -1551,8 +1559,6 @@ test diffHalfMatch { .expected = null, }}); - if (true) return error.SkipZigTest; // TODO - // Single matches try testing.checkAllAllocationFailures(testing.allocator, testDiffHalfMatch, .{.{ .dmp = one_timeout, From aedb7d60c0fc73edfb0a59b605d8c2a9d22c169b Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 16:44:46 -0400 Subject: [PATCH 067/176] Fixes two leaks --- DiffMatchPatch.zig | 98 +++++++++++++++++++++++----------------------- 1 file changed, 50 insertions(+), 48 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 3f6789e..daec121 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -862,13 +862,12 @@ fn diffCharsToLines( defer text.deinit(allocator); for (diffs) |*d| { - text.items.len = 0; var j: usize = 0; while (j < d.text.len) : (j += 1) { try text.appendSlice(allocator, line_array[d.text[j]]); } allocator.free(d.text); - d.text = try allocator.dupe(u8, text.items); + d.text = try text.toOwnedSlice(allocator); } } @@ -913,16 +912,15 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo { const ii = pointer - count_delete - count_insert - 1; var nt = try allocator.alloc(u8, diffs.items[ii].text.len + common_length); - const ot = diffs.items[ii].text; defer allocator.free(ot); @memcpy(nt[0..ot.len], ot); @memcpy(nt[ot.len..], text_insert.items[0..common_length]); diffs.items[ii].text = nt; } else { + try diffs.ensureUnusedCapacity(allocator, 1); const text = try allocator.dupe(u8, text_insert.items[0..common_length]); - errdefer allocator.free(text); - try diffs.insert(allocator, 0, Diff.init(.equal, text)); + diffs.insertAssumeCapacity(0, Diff.init(.equal, text)); pointer += 1; } try text_insert.replaceRange(allocator, 0, common_length, &.{}); @@ -933,11 +931,11 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo common_length = diffCommonSuffix(text_insert.items, text_delete.items); if (common_length != 0) { const old_text = diffs.items[pointer].text; - defer allocator.free(old_text); diffs.items[pointer].text = try std.mem.concat(allocator, u8, &.{ text_insert.items[text_insert.items.len - common_length ..], old_text, }); + defer allocator.free(old_text); text_insert.items.len -= common_length; text_delete.items.len -= common_length; } @@ -1004,38 +1002,38 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo { // This is a single edit surrounded by equalities. if (std.mem.endsWith(u8, diffs.items[pointer].text, diffs.items[pointer - 1].text)) { + const old_pt = diffs.items[pointer].text; const pt = try std.mem.concat(allocator, u8, &.{ diffs.items[pointer - 1].text, diffs.items[pointer].text[0 .. diffs.items[pointer].text.len - diffs.items[pointer - 1].text.len], }); + defer allocator.free(old_pt); + diffs.items[pointer].text = pt; + const old_pt1t = diffs.items[pointer + 1].text; const p1t = try std.mem.concat(allocator, u8, &.{ diffs.items[pointer - 1].text, diffs.items[pointer + 1].text, }); - const old_pt = diffs.items[pointer].text; - defer allocator.free(old_pt); - const old_pt1t = diffs.items[pointer + 1].text; defer allocator.free(old_pt1t); - diffs.items[pointer].text = pt; diffs.items[pointer + 1].text = p1t; freeRangeDiffList(allocator, diffs, pointer - 1, 1); try diffs.replaceRange(allocator, pointer - 1, 1, &.{}); changes = true; } else if (std.mem.startsWith(u8, diffs.items[pointer].text, diffs.items[pointer + 1].text)) { + const old_ptm1 = diffs.items[pointer - 1].text; const pm1t = try std.mem.concat(allocator, u8, &.{ diffs.items[pointer - 1].text, diffs.items[pointer + 1].text, }); + defer allocator.free(old_ptm1); + diffs.items[pointer - 1].text = pm1t; + const old_pt = diffs.items[pointer].text; + defer allocator.free(old_pt); const pt = try std.mem.concat(allocator, u8, &.{ diffs.items[pointer].text[diffs.items[pointer + 1].text.len..], diffs.items[pointer + 1].text, }); - const old_ptm1 = diffs.items[pointer - 1].text; - defer allocator.free(old_ptm1); - const old_pt = diffs.items[pointer].text; - defer allocator.free(old_pt); - diffs.items[pointer - 1].text = pm1t; diffs.items[pointer].text = pt; freeRangeDiffList(allocator, diffs, pointer + 1, 1); try diffs.replaceRange(allocator, pointer + 1, 1, &.{}); @@ -1777,14 +1775,16 @@ fn testDiffCharsToLines( } test diffCharsToLines { - if (true) return error.SkipZigTest; // TODO - // Convert chars up to lines. + var diff_list = DiffList{}; + defer deinitDiffList(testing.allocator, &diff_list); + try diff_list.ensureTotalCapacity(testing.allocator, 2); + diff_list.appendSliceAssumeCapacity(&.{ + Diff.init(.equal, try testing.allocator.dupe(u8, "\u{0001}\u{0002}\u{0001}")), + Diff.init(.insert, try testing.allocator.dupe(u8, "\u{0002}\u{0001}\u{0002}")), + }); try testing.checkAllAllocationFailures(testing.allocator, testDiffCharsToLines, .{.{ - .diffs = &.{ - .{ .operation = .equal, .text = "\u{0001}\u{0002}\u{0001}" }, - .{ .operation = .insert, .text = "\u{0002}\u{0001}\u{0002}" }, - }, + .diffs = diff_list.items, .line_array = &[_][]const u8{ "", "alpha\n", @@ -1900,8 +1900,6 @@ test diffCleanupMerge { }, }}); - if (true) return error.SkipZigTest; // TODO - // Prefix and suffix detection with equalities try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ .input = &.{ @@ -1933,17 +1931,19 @@ test diffCleanupMerge { }}); // Slide edit right - try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ - .input = &.{ - .{ .operation = .equal, .text = "c" }, - .{ .operation = .insert, .text = "ab" }, - .{ .operation = .equal, .text = "a" }, - }, - .expected = &.{ - .{ .operation = .equal, .text = "ca" }, - .{ .operation = .insert, .text = "ba" }, - }, - }}); + if (false) { // TODO #23 This test needs to dupe its data + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "c" }, + .{ .operation = .insert, .text = "ab" }, + .{ .operation = .equal, .text = "a" }, + }, + .expected = &.{ + .{ .operation = .equal, .text = "ca" }, + .{ .operation = .insert, .text = "ba" }, + }, + }}); + } // Slide edit left recursive try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ @@ -1960,20 +1960,22 @@ test diffCleanupMerge { }, }}); - // Slide edit right recursive - try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ - .input = &.{ - .{ .operation = .equal, .text = "x" }, - .{ .operation = .delete, .text = "ca" }, - .{ .operation = .equal, .text = "c" }, - .{ .operation = .delete, .text = "b" }, - .{ .operation = .equal, .text = "a" }, - }, - .expected = &.{ - .{ .operation = .equal, .text = "xca" }, - .{ .operation = .delete, .text = "cba" }, - }, - }}); + if (false) { // TODO #23 This test needs to dupe its data + // Slide edit right recursive + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "x" }, + .{ .operation = .delete, .text = "ca" }, + .{ .operation = .equal, .text = "c" }, + .{ .operation = .delete, .text = "b" }, + .{ .operation = .equal, .text = "a" }, + }, + .expected = &.{ + .{ .operation = .equal, .text = "xca" }, + .{ .operation = .delete, .text = "cba" }, + }, + }}); + } // Empty merge try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ From 460f047e24378f749cfd5c071bbe6d0244012650 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 17:06:37 -0400 Subject: [PATCH 068/176] More memory order bugs --- DiffMatchPatch.zig | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index daec121..e7cb9e5 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -627,6 +627,7 @@ fn diffBisect( // Diff took too long and hit the deadline or // number of diffs equals number of characters, no commonality at all. var diffs = DiffList{}; + errdefer deinitDiffList(allocator, &diffs); try diffs.ensureUnusedCapacity(allocator, 2); diffs.appendAssumeCapacity(Diff.init( .delete, @@ -1279,18 +1280,21 @@ pub fn diffCleanupSemanticLossless( if (!std.mem.eql(u8, diffs.items[pointer - 1].text, best_equality_1.items)) { // We have an improvement, save it back to the diff. if (best_equality_1.items.len != 0) { - allocator.free(diffs.items[pointer - 1].text); + const old_text = diffs.items[pointer - 1].text; diffs.items[pointer - 1].text = try allocator.dupe(u8, best_equality_1.items); + allocator.free(old_text); } else { const old_diff = diffs.orderedRemove(pointer - 1); allocator.free(old_diff.text); pointer -= 1; } - allocator.free(diffs.items[pointer].text); + const old_text1 = diffs.items[pointer].text; diffs.items[pointer].text = try allocator.dupe(u8, best_edit.items); + allocator.free(old_text1); if (best_equality_2.items.len != 0) { - allocator.free(diffs.items[pointer + 1].text); + const old_text2 = diffs.items[pointer + 1].text; diffs.items[pointer + 1].text = try allocator.dupe(u8, best_equality_2.items); + allocator.free(old_text2); } else { const old_diff = diffs.orderedRemove(pointer + 1); allocator.free(old_diff.text); @@ -2023,6 +2027,19 @@ fn testDiffCleanupSemanticLossless( try testing.expectEqualDeep(params.expected, diffs.items); } +fn sliceToDiffList(allocator: Allocator, diff_slice: []const Diff) !DiffList { + var diff_list = DiffList{}; + errdefer deinitDiffList(allocator, &diff_list); + try diff_list.ensureTotalCapacity(allocator, diff_slice.len); + for (diff_slice) |d| { + diff_list.appendAssumeCapacity(Diff.init( + d.operation, + try allocator.dupe(u8, d.text), + )); + } + return diff_list; +} + test diffCleanupSemanticLossless { // Null case try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemanticLossless, .{.{ @@ -2030,8 +2047,7 @@ test diffCleanupSemanticLossless { .expected = &[_]Diff{}, }}); - if (true) return error.SkipZigTest; // TODO - + //defer deinitDiffList(allocator, &diffs); try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemanticLossless, .{.{ .input = &.{ .{ .operation = .equal, .text = "AAA\r\n\r\nBBB" }, @@ -2179,8 +2195,6 @@ test diffBisect { }, }}); - if (true) return error.SkipZigTest; // TODO - // Timeout try testing.checkAllAllocationFailures(testing.allocator, testDiffBisect, .{.{ .dmp = this, From d5dff307f64a70585608a61747b065982b984012 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 17:13:58 -0400 Subject: [PATCH 069/176] Last of the skipped tests eliminated There are a few tests which need to have their data duped, to prevent a segfault. All clearly marked. --- DiffMatchPatch.zig | 94 +++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 48 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index e7cb9e5..be82049 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -131,7 +131,7 @@ fn diffInternal( // Check for equality (speedup). if (std.mem.eql(u8, before, after)) { var diffs = DiffList{}; - + errdefer deinitDiffList(allocator, &diffs); if (before.len != 0) { try diffs.ensureUnusedCapacity(allocator, 1); diffs.appendAssumeCapacity(Diff.init( @@ -2224,8 +2224,6 @@ fn testDiff( } test diff { - if (true) return error.SkipZigTest; // TODO - const this: DiffMatchPatch = .{ .diff_timeout = 0 }; // Null case. @@ -2597,8 +2595,6 @@ test diffCleanupSemantic { }, }}); - if (true) return error.SkipZigTest; // TODO - // Word boundaries try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ .input = &.{ @@ -2625,49 +2621,51 @@ test diffCleanupSemantic { }, }}); - // Overlap elimination - try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ - .input = &.{ - .{ .operation = .delete, .text = "abcxxx" }, - .{ .operation = .insert, .text = "xxxdef" }, - }, - .expected = &.{ - .{ .operation = .delete, .text = "abc" }, - .{ .operation = .equal, .text = "xxx" }, - .{ .operation = .insert, .text = "def" }, - }, - }}); + if (false) { // TODO #23 This test needs to dupe its data + // Overlap elimination + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ + .{ .operation = .delete, .text = "abcxxx" }, + .{ .operation = .insert, .text = "xxxdef" }, + }, + .expected = &.{ + .{ .operation = .delete, .text = "abc" }, + .{ .operation = .equal, .text = "xxx" }, + .{ .operation = .insert, .text = "def" }, + }, + }}); - // Reverse overlap elimination - try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ - .input = &.{ - .{ .operation = .delete, .text = "xxxabc" }, - .{ .operation = .insert, .text = "defxxx" }, - }, - .expected = &.{ - .{ .operation = .insert, .text = "def" }, - .{ .operation = .equal, .text = "xxx" }, - .{ .operation = .delete, .text = "abc" }, - }, - }}); + // Reverse overlap elimination + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ + .{ .operation = .delete, .text = "xxxabc" }, + .{ .operation = .insert, .text = "defxxx" }, + }, + .expected = &.{ + .{ .operation = .insert, .text = "def" }, + .{ .operation = .equal, .text = "xxx" }, + .{ .operation = .delete, .text = "abc" }, + }, + }}); - // Two overlap eliminations - try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ - .input = &.{ - .{ .operation = .delete, .text = "abcd1212" }, - .{ .operation = .insert, .text = "1212efghi" }, - .{ .operation = .equal, .text = "----" }, - .{ .operation = .delete, .text = "A3" }, - .{ .operation = .insert, .text = "3BC" }, - }, - .expected = &.{ - .{ .operation = .delete, .text = "abcd" }, - .{ .operation = .equal, .text = "1212" }, - .{ .operation = .insert, .text = "efghi" }, - .{ .operation = .equal, .text = "----" }, - .{ .operation = .delete, .text = "A" }, - .{ .operation = .equal, .text = "3" }, - .{ .operation = .insert, .text = "BC" }, - }, - }}); + // Two overlap eliminations + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ + .{ .operation = .delete, .text = "abcd1212" }, + .{ .operation = .insert, .text = "1212efghi" }, + .{ .operation = .equal, .text = "----" }, + .{ .operation = .delete, .text = "A3" }, + .{ .operation = .insert, .text = "3BC" }, + }, + .expected = &.{ + .{ .operation = .delete, .text = "abcd" }, + .{ .operation = .equal, .text = "1212" }, + .{ .operation = .insert, .text = "efghi" }, + .{ .operation = .equal, .text = "----" }, + .{ .operation = .delete, .text = "A" }, + .{ .operation = .equal, .text = "3" }, + .{ .operation = .insert, .text = "BC" }, + }, + }}); + } } From ddcbe8f38bf2ffb2a522734aa52ce0c8c3cf0226 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 19:36:22 -0400 Subject: [PATCH 070/176] Errdefer in rebuildtexts --- DiffMatchPatch.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index be82049..43f35cb 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2144,6 +2144,10 @@ fn rebuildtexts(allocator: std.mem.Allocator, diffs: DiffList) ![2][]const u8 { std.ArrayList(u8).init(allocator), std.ArrayList(u8).init(allocator), }; + errdefer { + allocator.free(text[0]); + allocator.free(text[1]); + } for (diffs.items) |myDiff| { if (myDiff.operation != .insert) { From 593d38f2f88d0bfe28286731e91c4988ca030b7f Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 20:03:57 -0400 Subject: [PATCH 071/176] Tests for rebuildtexts --- DiffMatchPatch.zig | 64 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 43f35cb..b2454e6 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2138,15 +2138,14 @@ test diffCleanupSemanticLossless { }}); } -/// TODO this function obviously leaks memory on error fn rebuildtexts(allocator: std.mem.Allocator, diffs: DiffList) ![2][]const u8 { var text = [2]std.ArrayList(u8){ std.ArrayList(u8).init(allocator), std.ArrayList(u8).init(allocator), }; errdefer { - allocator.free(text[0]); - allocator.free(text[1]); + text[0].deinit(); + text[1].deinit(); } for (diffs.items) |myDiff| { @@ -2163,6 +2162,65 @@ fn rebuildtexts(allocator: std.mem.Allocator, diffs: DiffList) ![2][]const u8 { }; } +fn testRebuildTexts(allocator: Allocator, diffs: DiffList, params: struct { + before: []const u8, + after: []const u8, +}) !void { + const texts = try rebuildtexts(allocator, diffs); + defer { + allocator.free(texts[0]); + allocator.free(texts[1]); + } + try testing.expectEqualStrings(params.before, texts[0]); + try testing.expectEqualStrings(params.after, texts[1]); +} + +test rebuildtexts { + { + var diffs = try sliceToDiffList(testing.allocator, &.{ + .{ .operation = .insert, .text = "abcabc" }, + .{ .operation = .equal, .text = "defdef" }, + .{ .operation = .delete, .text = "ghighi" }, + }); + defer deinitDiffList(testing.allocator, &diffs); + try testing.checkAllAllocationFailures(testing.allocator, testRebuildTexts, .{ + diffs, + .{ + .before = "defdefghighi", + .after = "abcabcdefdef", + }, + }); + } + { + var diffs = try sliceToDiffList(testing.allocator, &.{ + .{ .operation = .insert, .text = "xxx" }, + .{ .operation = .delete, .text = "yyy" }, + }); + defer deinitDiffList(testing.allocator, &diffs); + try testing.checkAllAllocationFailures(testing.allocator, testRebuildTexts, .{ + diffs, + .{ + .before = "yyy", + .after = "xxx", + }, + }); + } + { + var diffs = try sliceToDiffList(testing.allocator, &.{ + .{ .operation = .equal, .text = "xyz" }, + .{ .operation = .equal, .text = "pdq" }, + }); + defer deinitDiffList(testing.allocator, &diffs); + try testing.checkAllAllocationFailures(testing.allocator, testRebuildTexts, .{ + diffs, + .{ + .before = "xyzpdq", + .after = "xyzpdq", + }, + }); + } +} + fn testDiffBisect( allocator: std.mem.Allocator, params: struct { From 39b99aac99c72722b76547ff324ff8d7f9022baa Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 20:45:16 -0400 Subject: [PATCH 072/176] Last of the leaks --- DiffMatchPatch.zig | 167 ++++++++++++++++++++++----------------------- 1 file changed, 81 insertions(+), 86 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index b2454e6..3b863fd 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1009,14 +1009,14 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo diffs.items[pointer].text[0 .. diffs.items[pointer].text.len - diffs.items[pointer - 1].text.len], }); - defer allocator.free(old_pt); + allocator.free(old_pt); diffs.items[pointer].text = pt; const old_pt1t = diffs.items[pointer + 1].text; const p1t = try std.mem.concat(allocator, u8, &.{ diffs.items[pointer - 1].text, diffs.items[pointer + 1].text, }); - defer allocator.free(old_pt1t); + allocator.free(old_pt1t); diffs.items[pointer + 1].text = p1t; freeRangeDiffList(allocator, diffs, pointer - 1, 1); try diffs.replaceRange(allocator, pointer - 1, 1, &.{}); @@ -1027,14 +1027,14 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo diffs.items[pointer - 1].text, diffs.items[pointer + 1].text, }); - defer allocator.free(old_ptm1); + allocator.free(old_ptm1); diffs.items[pointer - 1].text = pm1t; const old_pt = diffs.items[pointer].text; - defer allocator.free(old_pt); const pt = try std.mem.concat(allocator, u8, &.{ diffs.items[pointer].text[diffs.items[pointer + 1].text.len..], diffs.items[pointer + 1].text, }); + allocator.free(old_pt); diffs.items[pointer].text = pt; freeRangeDiffList(allocator, diffs, pointer + 1, 1); try diffs.replaceRange(allocator, pointer + 1, 1, &.{}); @@ -1141,8 +1141,6 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError { // Overlap found. // Insert an equality and trim the surrounding edits. - defer allocator.free(deletion); - defer allocator.free(insertion); try diffs.ensureUnusedCapacity(allocator, 1); diffs.insertAssumeCapacity( @intCast(pointer), @@ -1153,8 +1151,10 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError ); diffs.items[@intCast(pointer - 1)].text = try allocator.dupe(u8, deletion[0 .. deletion.len - overlap_length1]); + allocator.free(deletion); diffs.items[@intCast(pointer + 1)].text = try allocator.dupe(u8, insertion[overlap_length1..]); + allocator.free(insertion); pointer += 1; } } else { @@ -1163,8 +1163,6 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError { // Reverse overlap found. // Insert an equality and swap and trim the surrounding edits. - defer allocator.free(deletion); - defer allocator.free(insertion); try diffs.ensureUnusedCapacity(allocator, 1); diffs.insertAssumeCapacity( @intCast(pointer), @@ -1173,11 +1171,14 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError try allocator.dupe(u8, deletion[0..overlap_length2]), ), ); - diffs.items[@intCast(pointer - 1)].operation = .insert; const new_minus = try allocator.dupe(u8, insertion[0 .. insertion.len - overlap_length2]); + errdefer allocator.free(new_minus); // necessary due to swap + const new_plus = try allocator.dupe(u8, deletion[overlap_length2..]); + allocator.free(deletion); + allocator.free(insertion); + diffs.items[@intCast(pointer - 1)].operation = .insert; diffs.items[@intCast(pointer - 1)].text = new_minus; diffs.items[@intCast(pointer + 1)].operation = .delete; - const new_plus = try allocator.dupe(u8, deletion[overlap_length2..]); diffs.items[@intCast(pointer + 1)].text = new_plus; pointer += 1; } @@ -1290,7 +1291,7 @@ pub fn diffCleanupSemanticLossless( } const old_text1 = diffs.items[pointer].text; diffs.items[pointer].text = try allocator.dupe(u8, best_edit.items); - allocator.free(old_text1); + defer allocator.free(old_text1); if (best_equality_2.items.len != 0) { const old_text2 = diffs.items[pointer + 1].text; diffs.items[pointer + 1].text = try allocator.dupe(u8, best_equality_2.items); @@ -1935,19 +1936,17 @@ test diffCleanupMerge { }}); // Slide edit right - if (false) { // TODO #23 This test needs to dupe its data - try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ - .input = &.{ - .{ .operation = .equal, .text = "c" }, - .{ .operation = .insert, .text = "ab" }, - .{ .operation = .equal, .text = "a" }, - }, - .expected = &.{ - .{ .operation = .equal, .text = "ca" }, - .{ .operation = .insert, .text = "ba" }, - }, - }}); - } + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "c" }, + .{ .operation = .insert, .text = "ab" }, + .{ .operation = .equal, .text = "a" }, + }, + .expected = &.{ + .{ .operation = .equal, .text = "ca" }, + .{ .operation = .insert, .text = "ba" }, + }, + }}); // Slide edit left recursive try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ @@ -1964,22 +1963,20 @@ test diffCleanupMerge { }, }}); - if (false) { // TODO #23 This test needs to dupe its data - // Slide edit right recursive - try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ - .input = &.{ - .{ .operation = .equal, .text = "x" }, - .{ .operation = .delete, .text = "ca" }, - .{ .operation = .equal, .text = "c" }, - .{ .operation = .delete, .text = "b" }, - .{ .operation = .equal, .text = "a" }, - }, - .expected = &.{ - .{ .operation = .equal, .text = "xca" }, - .{ .operation = .delete, .text = "cba" }, - }, - }}); - } + // Slide edit right recursive + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ + .input = &.{ + .{ .operation = .equal, .text = "x" }, + .{ .operation = .delete, .text = "ca" }, + .{ .operation = .equal, .text = "c" }, + .{ .operation = .delete, .text = "b" }, + .{ .operation = .equal, .text = "a" }, + }, + .expected = &.{ + .{ .operation = .equal, .text = "xca" }, + .{ .operation = .delete, .text = "cba" }, + }, + }}); // Empty merge try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupMerge, .{.{ @@ -2683,51 +2680,49 @@ test diffCleanupSemantic { }, }}); - if (false) { // TODO #23 This test needs to dupe its data - // Overlap elimination - try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ - .input = &.{ - .{ .operation = .delete, .text = "abcxxx" }, - .{ .operation = .insert, .text = "xxxdef" }, - }, - .expected = &.{ - .{ .operation = .delete, .text = "abc" }, - .{ .operation = .equal, .text = "xxx" }, - .{ .operation = .insert, .text = "def" }, - }, - }}); + // Overlap elimination + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ + .{ .operation = .delete, .text = "abcxxx" }, + .{ .operation = .insert, .text = "xxxdef" }, + }, + .expected = &.{ + .{ .operation = .delete, .text = "abc" }, + .{ .operation = .equal, .text = "xxx" }, + .{ .operation = .insert, .text = "def" }, + }, + }}); - // Reverse overlap elimination - try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ - .input = &.{ - .{ .operation = .delete, .text = "xxxabc" }, - .{ .operation = .insert, .text = "defxxx" }, - }, - .expected = &.{ - .{ .operation = .insert, .text = "def" }, - .{ .operation = .equal, .text = "xxx" }, - .{ .operation = .delete, .text = "abc" }, - }, - }}); - - // Two overlap eliminations - try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ - .input = &.{ - .{ .operation = .delete, .text = "abcd1212" }, - .{ .operation = .insert, .text = "1212efghi" }, - .{ .operation = .equal, .text = "----" }, - .{ .operation = .delete, .text = "A3" }, - .{ .operation = .insert, .text = "3BC" }, - }, - .expected = &.{ - .{ .operation = .delete, .text = "abcd" }, - .{ .operation = .equal, .text = "1212" }, - .{ .operation = .insert, .text = "efghi" }, - .{ .operation = .equal, .text = "----" }, - .{ .operation = .delete, .text = "A" }, - .{ .operation = .equal, .text = "3" }, - .{ .operation = .insert, .text = "BC" }, - }, - }}); - } + // Reverse overlap elimination + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ + .{ .operation = .delete, .text = "xxxabc" }, + .{ .operation = .insert, .text = "defxxx" }, + }, + .expected = &.{ + .{ .operation = .insert, .text = "def" }, + .{ .operation = .equal, .text = "xxx" }, + .{ .operation = .delete, .text = "abc" }, + }, + }}); + + // Two overlap eliminations + try testing.checkAllAllocationFailures(testing.allocator, testDiffCleanupSemantic, .{.{ + .input = &.{ + .{ .operation = .delete, .text = "abcd1212" }, + .{ .operation = .insert, .text = "1212efghi" }, + .{ .operation = .equal, .text = "----" }, + .{ .operation = .delete, .text = "A3" }, + .{ .operation = .insert, .text = "3BC" }, + }, + .expected = &.{ + .{ .operation = .delete, .text = "abcd" }, + .{ .operation = .equal, .text = "1212" }, + .{ .operation = .insert, .text = "efghi" }, + .{ .operation = .equal, .text = "----" }, + .{ .operation = .delete, .text = "A" }, + .{ .operation = .equal, .text = "3" }, + .{ .operation = .insert, .text = "BC" }, + }, + }}); } From 962b461529bbccb1f2e966e1569531bd1d4e8dfd Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 7 Jul 2024 23:59:18 -0400 Subject: [PATCH 073/176] I am once again patching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ¯\_(ツ)_/¯¯\_(ツ)_/¯¯\_(ツ)_/¯¯\_(ツ)_/¯¯\_(ツ)_/¯¯\_(ツ)_/¯¯\_(ツ)_/¯ --- DiffMatchPatch.zig | 1798 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 1739 insertions(+), 59 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 3b863fd..5241819 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2,9 +2,51 @@ const DiffMatchPatch = @This(); const std = @import("std"); const testing = std.testing; +const assert = std.debug.assert; const Allocator = std.mem.Allocator; const ArrayListUnmanaged = std.ArrayListUnmanaged; const DiffList = ArrayListUnmanaged(Diff); +const PatchList = ArrayListUnmanaged(Patch); + +pub const DiffError = error{ + OutOfMemory, + BadPatchString, +}; + +//| XXX This boolean is entirely for calming the compiler down while working + +const XXX = false; + +//| Fields + +/// Number of milliseconds to map a diff before giving up (0 for infinity). +diff_timeout: u64 = 1000, +/// Cost of an empty edit operation in terms of edit characters. +diff_edit_cost: u16 = 4, + +/// At what point is no match declared (0.0 = perfection, 1.0 = very loose). +/// This defaults to 0.05, on the premise that the library will mostly be +/// used in cases where failure is better than a bad patch application. +match_threshold: f32 = 0.05, + +/// How far to search for a match (0 = exact location, 1000+ = broad match). +/// A match this many characters away from the expected location will add +/// 1.0 to the score (0.0 is a perfect match). +match_distance: u32 = 1000, + +/// The number of bits in a usize. +match_max_bits: u8 = @bitSizeOf(usize), + +/// When deleting a large block of text (over ~64 characters), how close +/// do the contents have to be to match the expected contents. (0.0 = +/// perfection, 1.0 = very loose). Note that Match_Threshold controls +/// how closely the end points of a delete need to match. +patch_delete_threshold: f32 = 0.5, + +/// Chunk size for context length. +patch_margin: u8 = 4, + +//| Allocation Management Helpers /// Deinit an `ArrayListUnmanaged(Diff)` and the allocated slices of /// text in each `Diff`. @@ -15,6 +57,8 @@ pub fn deinitDiffList(allocator: Allocator, diffs: *DiffList) void { } } +/// Free a range of Diffs inside a list. Used during cleanups and +/// edits. fn freeRangeDiffList( allocator: Allocator, diffs: *DiffList, @@ -31,6 +75,8 @@ fn freeRangeDiffList( /// DMP with default configuration options pub const default = DiffMatchPatch{}; +/// Represents a single edit operation. +/// TODO rename this Edit pub const Diff = struct { pub const Operation = enum { insert, @@ -59,42 +105,108 @@ pub const Diff = struct { return a.operation == b.operation and std.mem.eql(u8, a.text, b.text); } - test eql { - const equal_a: Diff = .{ .operation = .equal, .text = "a" }; - const insert_a: Diff = .{ .operation = .insert, .text = "a" }; - const equal_b: Diff = .{ .operation = .equal, .text = "b" }; - const delete_b: Diff = .{ .operation = .delete, .text = "b" }; - - try testing.expect(equal_a.eql(equal_a)); - try testing.expect(!insert_a.eql(equal_a)); - try testing.expect(!equal_a.eql(equal_b)); - try testing.expect(!equal_a.eql(delete_b)); + pub fn clone(self: Diff, allocator: Allocator) !Diff { + return Diff{ + .operation = self.operation, + .text = try allocator.dupe(u8, self.text), + }; } }; -/// Number of milliseconds to map a diff before giving up (0 for infinity). -diff_timeout: u64 = 1000, -/// Cost of an empty edit operation in terms of edit characters. -diff_edit_cost: u16 = 4, +pub const Patch = struct { + /// Diffs to be applied + diffs: DiffList, // TODO This should be a Diff + /// Start of patch in before text + start1: usize = 0, + length1: usize = 0, + /// Start of patch in after text + start2: usize = 0, + length2: usize = 0, + + pub fn toString(patch: Patch) ![]const u8 { + // TODO + _ = patch; + } -/// At what point is no match declared (0.0 = perfection, 1.0 = very loose). -match_threshold: f32 = 0.5, -/// How far to search for a match (0 = exact location, 1000+ = broad match). -/// A match this many characters away from the expected location will add -/// 1.0 to the score (0.0 is a perfect match). -match_distance: u32 = 1000, -/// The number of bits in an int. -match_max_bits: u16 = 32, + pub fn writeTo(writer: anytype) !usize { + // TODO + _ = writer; + } -/// When deleting a large block of text (over ~64 characters), how close -/// do the contents have to be to match the expected contents. (0.0 = -/// perfection, 1.0 = very loose). Note that Match_Threshold controls -/// how closely the end points of a delete need to match. -patch_delete_threshold: f32 = 0.5, -/// Chunk size for context length. -patch_margin: u16 = 4, + /// Make a clone of the Patch, including all Diffs. + pub fn clone(patch: Patch, allocator: Allocator) !Patch { + var new_diffs = DiffList{}; + new_diffs.initCapacity(allocator, patch.diffs.items.len); + for (patch.diffs) |a_diff| { + try new_diffs.append(try a_diff.clone(allocator)); + } + return Patch{ + .diffs = new_diffs, + .start1 = patch.start1, + .length1 = patch.length1, + .start2 = patch.start2, + .length2 = patch.length2, + }; + } -pub const DiffError = error{OutOfMemory}; + pub fn deinit(patch: *Patch, allocator: Allocator) void { + deinitDiffList(allocator, patch.diffs); + } + + /// Emit patch in Unidiff format, as specifified here: + /// https://github.com/google/diff-match-patch/wiki/Unidiff + /// This is similar to GNU Unidiff format, but not identical. + /// Header: @@ -382,8 +481,9 @@ + /// Indices are printed as 1-based, not 0-based. + /// @return The GNU diff string. + pub fn asText(patch: Patch, allocator: Allocator) ![]const u8 { + var text_array = std.ArrayList(u8).init(allocator); + defer text_array.deinit(); + const writer = text_array.writer(); + try patch.writeText(writer, patch); + return text_array.toOwnedSlice(); + } + + const format = std.fmt.format; + + /// Stream textual patch representation to Writer. See `asText` + /// for more information. + pub fn writeText(writer: anytype, patch: Patch) !void { + // Write header. + _ = try writer.write(PATCH_HEAD); + // Stream coordinates + if (patch.length1 == 0) { + try format(writer, "{d},0", .{patch.start1}); + } else if (patch.length1 == 1) { + try format(writer, "{d}", .{patch.start1 + 1}); + } else { + try format(writer, "{d},{d}", .{ patch.start1 + 1, patch.length1 }); + } + _ = try writer.write(" +"); + if (patch.length2 == 0) { + try std.fmt.format(writer, "{d},0", .{patch.start2}); + } else if (patch.length2 == 1) { + _ = try format(writer, "{d}", .{patch.start2 + 1}); + } else { + try format(writer, "{d},{d}", .{ patch.start2 + 1, patch.length2 }); + } + _ = writer.write(PATCH_TAIL); + // Escape the body of the patch with %xx notation. + for (patch.diffs) |a_diff| { + switch (a_diff.operation) { + .insert => try writer.writeByte('+'), + .delete => try writer.writeByte('-'), + .equal => try writer.writeByte('='), + } + _ = try writeUriEncoded(writer, diff.text); + try writer.writeByte('\n'); + } + return; + } +}; + +const PATCH_HEAD = "@@ -"; +const PATCH_TAIL = " @@\n"; /// Find the differences between two texts. /// @param before Old string to be diffed. @@ -159,7 +271,6 @@ fn diffInternal( errdefer deinitDiffList(allocator, &diffs); // Restore the prefix and suffix. - if (common_prefix.len != 0) { try diffs.ensureUnusedCapacity(allocator, 1); diffs.insertAssumeCapacity(0, Diff.init( @@ -179,26 +290,70 @@ fn diffInternal( return diffs; } +/// Test if a byte is a UTF-8 follow byte +inline fn is_follow(byte: u8) bool { + return byte & 0b1100_0000 == 0b1000_0000; +} + +/// Find a common prefix which respects UTF-8 code point boundaries. fn diffCommonPrefix(before: []const u8, after: []const u8) usize { const n = @min(before.len, after.len); var i: usize = 0; - while (i < n) : (i += 1) { - if (before[i] != after[i]) { - return i; + var b = before[i]; + const a = after[i]; + if (a != b) { + if (is_follow(a) and is_follow(b)) { + // We've clipped a codepoint, back out + if (i == 0) return i; // Malformed UTF-8 is always possible + i -= 1; + // We'll track `before` since they must be the same: + b = before[i]; + assert(b == after[i]); + while (i != 0 and is_follow(b)) { + i -= 1; + b = before[i]; + assert(b == after[i]); + } + // Now we're either at zero, or at the lead: + return i; + } else { + return i; + } } } return n; } +/// Find a common suffix which respects UTF-8 code point boundaries fn diffCommonSuffix(before: []const u8, after: []const u8) usize { const n = @min(before.len, after.len); var i: usize = 1; - + var was_follow = false; while (i <= n) : (i += 1) { - if (before[before.len - i] != after[after.len - i]) { - return i - 1; + var b = before[before.len - i]; + const a = after[after.len - i]; + if (a != b) { + if (was_follow) { + // Means we're at at least 2: + assert(i > 1); + // We just saw an identical follow byte, so we back + // out forward: + i -= 1; + b = before[before.len - i]; + assert(b == after[after.len - i]); + while (i > 1 and is_follow(b)) { + i -= 1; + b = before[before.len - i]; + assert(b == after[after.len - i]); + } // Either at one, or no more follow bytes: + return i - 1; + } else { + return i - 1; + } + } else { + was_follow = is_follow(b); // no need to check twice } } @@ -329,7 +484,6 @@ fn diffCompute( if (check_lines and before.len > 100 and after.len > 100) { return dmp.diffLineMode(allocator, before, after, deadline); } - return dmp.diffBisect(allocator, before, after, deadline); } @@ -365,7 +519,7 @@ fn diffHalfMatch( before: []const u8, after: []const u8, ) DiffError!?HalfMatchResult { - if (dmp.diff_timeout <= 0) { + if (dmp.diff_timeout == 0) { // Don't risk returning a non-optimal diff if we have unlimited time. return null; } @@ -556,11 +710,14 @@ fn diffBisect( x1 = v1.items[@intCast(k1_offset - 1)] + 1; } var y1 = x1 - k1; - while (x1 < before_length and - y1 < after_length and before[@intCast(x1)] == after[@intCast(y1)]) - { - x1 += 1; - y1 += 1; + while (x1 < before_length and y1 < after_length) { + const match, const d1 = equalForward(before, after, x1, y1); + if (match) { + x1 += d1; + y1 += d1; + } else { + break; + } } v1.items[@intCast(k1_offset)] = x1; if (x1 > before_length) { @@ -595,12 +752,19 @@ fn diffBisect( x2 = v2.items[@intCast(k2_offset - 1)] + 1; } var y2: isize = x2 - k2; - while (x2 < before_length and y2 < after_length and - before[@intCast(before_length - x2 - 1)] == - after[@intCast(after_length - y2 - 1)]) - { - x2 += 1; - y2 += 1; + while (x2 < before_length and y2 < after_length) { + const match, const d1 = equalBackward( + before, + after, + before_length - x2 - 1, + after_length - y2 - 1, + ); + if (match) { + x2 += d1; + y2 += d1; + } else { + break; + } } v2.items[@intCast(k2_offset)] = x2; if (x2 > before_length) { @@ -640,6 +804,111 @@ fn diffBisect( return diffs; } +/// Match up to a full character in the forward direction. Note the +/// goal here: we aren't validating Unicode, we're making sure we don't +/// split code unit sequences. We might get non-minimal diffs on bad +/// UTF-8, but that's fine. +fn equalForward( + before: []const u8, + after: []const u8, + b_i: isize, + a_i: isize, +) struct { bool, isize } { + const b_u: usize = @intCast(b_i); + const a_u: usize = @intCast(a_i); + const b1c = before[b_u]; + const a1c = after[a_u]; + if (b1c == a1c) { + // how many codeunits might we expect? + // ASCII is easy: + if (b1c < 0x80) { + return .{ true, 1 }; + } else { + switch (b1c) { + 0xc2...0xdf => { + // two bytes + if (b_u + 1 >= before.len or a_u + 1 >= after.len) { + // it's a match ¯\_(ツ)_/¯ + return .{ true, 1 }; + } // length is unused for false results + return .{ before[b_u + 1] == after[a_u + 1], 2 }; + }, + 0xe0...0xef => { + // three bytes + if (b_u + 2 >= before.len or a_u + 2 >= after.len) { + return .{ true, 1 }; + } + const m2 = before[b_u + 1] == after[a_u + 1]; + const m3 = before[b_u + 2] == after[a_u + 2]; + return .{ m2 and m3, 3 }; + }, + 0xf0...0xf4 => { + // four bytes + if (b_u + 3 >= before.len or a_u + 3 >= after.len) { + return .{ true, 1 }; + } + const m = same: { + const m2 = before[b_u + 1] == after[a_u + 1]; + const m3 = before[b_u + 2] == after[a_u + 2]; + const m4 = before[b_u + 3] == after[a_u + 3]; + break :same m2 and m3 and m4; + }; + return .{ m, 4 }; + }, // follow byte or invalid high, doesn't matter, match + else => return .{ true, 1 }, + } + } + } else { + return .{ false, 0 }; + } +} + +/// Match characters backward, avoiding splitting two valid codeunits with a +/// common suffix. Once again, we are not interested in validating the text, +/// just in preventing a spurious diff which truncates Unicode. +fn equalBackward( + before: []const u8, + after: []const u8, + b_i: isize, + a_i: isize, +) struct { bool, isize } { + const b_u: usize = @intCast(b_i); + const a_u: usize = @intCast(a_i); + const b1c = before[b_u]; + const a1c = after[a_u]; + if (b1c == a1c) { + // how many codeunits might we expect? + // different jam here! We have to match back to a lead: + switch (b1c) { + // follow byte might be a code unit sequence + 0x80...0xbf => { + // I'd rather double the offsets then deal with + // casting. Feel free to optimize... + var off: usize = 1; + var offi: isize = @intCast(off); + while (off < 4 and b_i - offi >= 0 and a_i - offi >= 0) { + const b = before[b_u - off]; + if (b != after[b_u - off]) { + // whole thing is a fail + return .{ false, 0 }; // here the offset doesn't matter + } + // check for lead byte + // since we presume well-formedness, any lead will do + if (0xc1 < b and b < 0xf5) { + return .{ true, offi + 1 }; + } + off += 1; + offi += 1; + } // since we didn't spot a plausible character, match 1 + return .{ true, 1 }; + }, // ASCII, malformed, don't care, + else => return .{ true, 1 }, + } + } else { + return .{ false, 0 }; + } +} + /// Given the location of the 'middle snake', split the diff in two parts /// and recurse. /// @param text1 Old string to be diffed. @@ -905,12 +1174,12 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo // Upon reaching an equality, check for prior redundancies. if (count_delete + count_insert > 1) { if (count_delete != 0 and count_insert != 0) { - // Factor out any common prefixies. + // Factor out any common prefixes. common_length = diffCommonPrefix(text_insert.items, text_delete.items); if (common_length != 0) { if ((pointer - count_delete - count_insert) > 0 and diffs.items[pointer - count_delete - count_insert - 1].operation == .equal) - { + { // The prefix is not at the start of the diffs const ii = pointer - count_delete - count_insert - 1; var nt = try allocator.alloc(u8, diffs.items[ii].text.len + common_length); const ot = diffs.items[ii].text; @@ -967,8 +1236,7 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo pointer += 1; } else if (pointer != 0 and diffs.items[pointer - 1].operation == .equal) { // Merge this equality with the previous one. - // TODO: Fix using realloc or smth - // Note: can't use realloc because the text is const + // Diff texts are []const u8 so a realloc isn't practical here var nt = try allocator.alloc(u8, diffs.items[pointer - 1].text.len + diffs.items[pointer].text.len); const ot = diffs.items[pointer - 1].text; defer (allocator.free(ot)); @@ -1362,7 +1630,7 @@ fn diffCleanupSemanticScore(one: []const u8, two: []const u8) usize { } /// Reduce the number of edits by eliminating operationally trivial -/// equalities. +/// equalities. TODO this needs tests pub fn diffCleanupEfficiency( dmp: DiffMatchPatch, allocator: std.mem.Allocator, @@ -1383,7 +1651,7 @@ pub fn diffCleanupEfficiency( var post_ins = false; // Is there a deletion operation after the last equality. var post_del = false; - while (pointer < diffs.Count) { + while (pointer < diffs.len) { if (diffs.items[pointer].operation == .equal) { // Equality found. if (diffs.items[pointer].text.len < dmp.diff_edit_cost and (post_ins or post_del)) { // Candidate found. @@ -1486,10 +1754,10 @@ fn diffCommonOverlap(text1_in: []const u8, text2_in: []const u8) usize { // Performance analysis: https://neil.fraser.name/news/2010/11/04/ var best: usize = 0; var length: usize = 1; - while (true) { + const best_idx = idx: while (true) { const pattern = text1[text_length - length ..]; const found = std.mem.indexOf(u8, text2, pattern) orelse - return best; + break :idx best; length += found; @@ -1497,11 +1765,1423 @@ fn diffCommonOverlap(text1_in: []const u8, text2_in: []const u8) usize { best = length; length += 1; } + }; + if (best_idx == 0) return best_idx; + // This would mean a truncation: lead or follow, followed by a follow + // which differs (or it would be included in our overlap) + if (text2[best_idx] >= 0x80 and is_follow(text2[best_idx + 1])) { + // back out + assert(best_idx == best); + if (!is_follow(text2[best])) { + // It's a lead, one back is fine + return best - 1; + } + best -= 1; + if (best == 0) return 0; + // It's ok to get no overlap, so we ignore malformation: + // a bunch of follows could walk back to zero, and that's + // fine with us + while (is_follow(text2[best])) { + best -= 1; + if (best == 0) return 0; + } + // should be a lead, but ASCII is fine, so + if (text2[best] < 0x80) { + return best; + } else { + return best - 1; + } + } + return best_idx; +} + +/// loc is a location in text1, compute and return the equivalent location in +/// text2. +/// e.g. "The cat" vs "The big cat", 1->1, 5->8 +/// @param diffs List of Diff objects. +/// @param loc Location within text1. +/// @return Location within text2. +/// +pub fn diffIndex(diffs: DiffList, loc: usize) usize { + // int chars1 = 0; + // int chars2 = 0; + // int last_chars1 = 0; + // int last_chars2 = 0; + var chars1: usize = 0; + var chars2: usize = 0; + var last_chars1: usize = 0; + var last_chars2: usize = 0; + // Dummy diff + var last_diff: Diff = Diff{ .operation = .equal, .text = "" }; + for (diffs) |a_diff| { + if (a_diff.operation != .insert) { + // Equality or deletion. + chars1 += a_diff.text.len; + } + if (a_diff.operation != .delete) { + // Equality or insertion. + chars2 += a_diff.text.len; + } + if (chars1 > loc) { + // Overshot the location. + last_diff = a_diff; + break; + } + } + last_chars1 = chars1; + last_chars2 = chars2; + + if (last_diff.text.len != 0 and last_diff.operation == .delete) { + // The location was deleted. + return last_chars2; + } + // Add the remaining character length. + return last_chars2 + (loc - last_chars1); +} + +/// A struct holding bookends for `diffPrittyFormat(diffs)`. +/// +/// May include a function taking an allocator and the diff, +/// which shall return the text of the diff, appropriately munged. +/// Note that if the function is provided, all text returned will +/// be freed, so it should always return a copy whether or not +/// edits are needed. +pub const DiffDecorations = struct { + delete_start: []const u8 = "", + delete_end: []const u8 = "", + insert_start: []const u8 = "", + insert_end: []const u8 = "", + equals_start: []const u8 = "", + equals_end: []const u8 = "", + pre_process: ?fn (Allocator, Diff) error{OutOfMemory}![]const u8 = null, +}; + +/// Decorations for classic Xterm printing: red for delete and +/// green for insert. +pub const xterm_classic = DiffDecorations{ + .delete_start = "\x1b[91m", + .delete_end = "\x1b[m", + .insert_start = "\x1b[92m", + .insert_end = "\x1b[m", +}; + +/// Return text representing a pretty-formatted `DiffList`. +/// See `DiffDecorations` for how to customize this output. +pub fn diffPrettyFormat( + allocator: Allocator, + diffs: DiffList, + deco: DiffDecorations, +) ![]const u8 { + var out = ArrayListUnmanaged(u8){}; + defer out.deinit(allocator); + const writer = out.writer(); + _ = try writeDiffPrettyFormat(allocator, writer, diffs, deco); + return out.toOwnedSlice(allocator); +} + +/// Write a pretty-formatted `DiffList` to `writer`. The `Allocator` +/// is only used if a custom text formatter is defined for +/// `DiffDecorations`. Returns number of bytes written. +pub fn writeDiffPrettyFormat( + allocator: Allocator, + writer: anytype, + diffs: DiffList, + deco: DiffDecorations, +) !usize { + var written: usize = 0; + for (diffs) |d| { + const text = if (deco.pre_process) |lambda| + try lambda(allocator, d) + else + d.text; + defer { + if (deco.pre_process) |_| + allocator.free(text); + } + switch (d.operation) { + .delete => { + // + written += try writer.write(deco.delete_start); + written += try writer.write(text); + written += try writer.write(deco.delete_end); + }, + .insert => { + written += try writer.write(deco.insert_start); + written += try writer.write(text); + written += try writer.write(deco.insert_end); + }, + .equals => { + written += try writer.write(deco.equals_start); + written += try writer.write(text); + written += try writer.write(deco.equals_end); + }, + } + } + return written; +} + +/// +/// Compute and return the source text (all equalities and deletions). +/// @param diffs List of `Diff` objects. +/// @return Source text. +/// +pub fn diffBeforeText(allocator: Allocator, diffs: DiffList) ![]const u8 { + var chars = ArrayListUnmanaged(u8){}; + defer chars.deinit(allocator); + for (diffs) |d| { + if (d.operation != .insert) { + try chars.appendSlice(allocator, d.text); + } + } + return chars.toOwnedSlice(allocator); +} + +/// +/// Compute and return the destination text (all equalities and insertions). +/// @param diffs List of `Diff` objects. +/// @return Destination text. +/// +pub fn diffAfterText(allocator: Allocator, diffs: DiffList) ![]const u8 { + var chars = ArrayListUnmanaged(u8){}; + defer chars.deinit(allocator); + for (diffs) |d| { + if (d.operation != .delete) { + try chars.appendSlice(allocator, d.text); + } + } + return chars.toOwnedSlice(allocator); +} + +/// +/// Compute the Levenshtein distance; the number of inserted, +/// deleted or substituted characters. +/// +/// @param diffs List of Diff objects. +/// @return Number of changes. +/// +pub fn diffLevenshtein(diffs: DiffList) usize { + var inserts: usize = 0; + var deletes: usize = 0; + var levenshtein: usize = 0; + for (diffs) |d| { + switch (d.operation) { + .insert => { + inserts += d.text.len; + }, + .delete => { + deletes += d.text.len; + }, + .equal => { + // A deletion and an insertion is one substitution. + levenshtein = @max(inserts, deletes); + inserts = 0; + deletes = 0; + }, + } + } + + return levenshtein + @max(inserts, deletes); +} + +//| MATCH FUNCTIONS + +/// Locate the best instance of 'pattern' in 'text' near 'loc'. +/// Returns -1 if no match found. +/// @param text The text to search. +/// @param pattern The pattern to search for. +/// @param loc The location to search around. +/// @return Best match index or -1. +pub fn matchMain(allocator: Allocator, text: []const u8, pattern: []const u8, passed_loc: usize) ?usize { + // Clamp the loc to fit within text. + const loc = @min(passed_loc, text.len); + if (std.mem.eql(u8, text, pattern)) { + // Shortcut (potentially not guaranteed by the algorithm) + // TODO would be good to know what the above means... + return 0; + } else if (text.len == 0) { + // Nothing to match. + return null; + } else if (loc + pattern.len <= text.len and std.mem.eql(u8, text[loc..pattern.length], pattern)) { + // Perfect match at the perfect spot! (Includes case of null pattern) + return loc; + } else { + // Do a fuzzy compare. + return matchBitap(allocator, text, pattern, loc); + } +} + +// TODO doubling the bits to fit in usize is nice and all, but there's no +// reason to be limited to that, we have bitsets which can be as large as +// we'd like. This could be passed a comptime power-of-two size, and use +// that to make an ArrayBitSet specialized for several sizes, up to, IDK, +// 2k? Then split very large patches only. 64, 256, 512, 1024, 2028, is +// a nice balance between code size and versatility. +// Something like this: +fn matchBitapImproved( + allocator: Allocator, + text: []const u8, + pattern: []const u8, + loc: usize, + UIntType: type, +) ?usize { + assert(pattern.len < @bitSizeOf(UIntType)); + const ShiftWidth = ShiftSizeForType(UIntType); + // Initialise the alphabet. + var map = try matchAlphabet(allocator, pattern); + defer map.deinit(); + // Highest score beyond which we give up. + var threshold = @This().threshold; + // Is there a nearby exact match? (speedup) + var best_loc = std.mem.indexOfPos(u8, text, pattern); + if (best_loc) |best| { + threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + } + // What about in the other direction? (speedup) + const trunc_text = text[0..@min(loc + pattern.len, text.len)]; + best_loc = std.mem.lastIndexOf(u8, trunc_text, pattern); + if (best_loc) |best| { + threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + } + // Initialise the bit arrays. + const shift: ShiftWidth = @intCast(pattern.len - 1); + // 0 for a match for faster bit twiddles + const matchmask = ~(1 << shift); + best_loc = null; + var bin_min: usize = undefined; + var bin_mid: usize = undefined; + var bin_max = pattern.len + text.len; + // null last_rd to simplying freeing memory + var last_rd = try allocator.alloc(UIntType, 0); + for (0..pattern.len) |d| { + // Scan for the best match; each iteration allows for one more error. + // Run a binary search to determine how far from 'loc' we can stray at + // this error level. + bin_min = 0; + bin_mid = bin_max; + while (bin_min < bin_mid) { + if (matchBitapScore(d, loc + bin_mid, loc, pattern) <= threshold) { + bin_min = bin_mid; + } else { + bin_max = bin_mid; + } + bin_mid = (bin_max - bin_min) / 2 + bin_min; + } + // Use the result from this iteration as the maximum for the next. + bin_max = bin_mid; + var start = @max(1, loc - bin_mid + 1); + const finish = @min(loc + bin_mid, text.len) + pattern.len; + var rd = try allocator.alloc(UIntType, finish + 2); + const dshift: ShiftWidth = @intCast(d); + rd[finish + 1] = 1 << dshift; + var j = finish; + while (j >= start) : (j -= 1) { + const char_match: usize = if (text.len <= j - 1 or !map.contains(text[j - 1])) + // Out of range. + 0 + else + map.get(text[j - 1]); + if (d == 0) { + // First pass: exact match. + rd[j] = ((rd[j + 1] << 1)) & char_match; + } else { + // Subsequent passes: fuzzy match. + rd[j] = ((rd[j + 1] << 1)) & char_match & (((last_rd[j + 1] & last_rd[j]) << 1)) & last_rd[j + 1]; + } + if ((rd[j] & matchmask) != 0) { + const score = matchBitapScore(d, j - 1, loc, pattern); + // This match will almost certainly be better than any existing + // match. But check anyway. + if (score <= threshold) { + // Told you so. + threshold = score; + best_loc = j - 1; + if (best_loc > loc) { + // When passing loc, don't exceed our current distance from loc. + start = @max(1, 2 * loc - best_loc); + } else { + // Already passed loc, downhill from here on in. + break; + } + } + } + } + if (matchBitapScore(d + 1, loc, loc, pattern) > threshold) { + // No hope for a (better) match at greater error levels. + break; + } + allocator.free(last_rd); + last_rd = rd; + } + allocator.free(last_rd); + return best_loc; +} + +fn ShiftSizeForType(T: type) type { + return switch (@typeInfo(T.Int.bits)) { + 64 => u6, + 256 => u8, + 1024 => u9, + 2048 => u10, + else => unreachable, + }; +} + +/// Locate the best instance of `pattern` in `text` near `loc` using the +/// Bitap algorithm. Returns -1 if no match found. +/// +/// @param text The text to search. +/// @param pattern The pattern to search for. +/// @param loc The location to search around. +/// @return Best match index or -1. +fn matchBitap( + allocator: Allocator, + text: []const u8, + pattern: []const u8, + loc: usize, +) !?usize { + // TODO decide what to do here: + // assert (Match_MaxBits == 0 || pattern.Length <= Match_MaxBits) + // : "Pattern too long for this application."; + + // Initialise the alphabet. + var map = try matchAlphabet(allocator, pattern); + defer map.deinit(); + // Highest score beyond which we give up. + var threshold = @This().threshold; + // Is there a nearby exact match? (speedup) + var best_loc = std.mem.indexOfPos(u8, text, pattern); + if (best_loc) |best| { + threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); } + // TODO obviously if we want a speedup here, we do this: + // if (threshold == 0.0) return best_loc; + // We don't have to unwrap best_loc because the retval is ?usize already + // What about in the other direction? (speedup) + const trunc_text = text[0..@min(loc + pattern.len, text.len)]; + best_loc = std.mem.lastIndexOf(u8, trunc_text, pattern); + if (best_loc) |best| { + threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + } + // Initialise the bit arrays. + const shift: u6 = @intCast(pattern.len - 1); + const matchmask = 1 << shift; + best_loc = null; + var bin_min: usize = undefined; + var bin_mid: usize = undefined; + var bin_max = pattern.len + text.len; + // null last_rd to simplying freeing memory + var last_rd: []usize = try allocator.alloct(usize, 0); + for (0..pattern.len) |d| { + // Scan for the best match; each iteration allows for one more error. + // Run a binary search to determine how far from 'loc' we can stray at + // this error level. + bin_min = 0; + bin_mid = bin_max; + while (bin_min < bin_mid) { + if (matchBitapScore(d, loc + bin_mid, loc, pattern) <= threshold) { + bin_min = bin_mid; + } else { + bin_max = bin_mid; + } + bin_mid = (bin_max - bin_min) / 2 + bin_min; + } + // Use the result from this iteration as the maximum for the next. + bin_max = bin_mid; + var start = @max(1, loc - bin_mid + 1); + const finish = @min(loc + bin_mid, text.len) + pattern.len; + var rd: []usize = allocator.alloc(usize, finish + 2); + const dshift: u6 = @intCast(d); + rd[finish + 1] = (1 << dshift) - 1; + var j = finish; + while (j >= start) : (j -= 1) { + const char_match: usize = if (text.len <= j - 1 or !map.contains(text[j - 1])) + // Out of range. + 0 + else + map.get(text[j - 1]); + if (d == 0) { + // First pass: exact match. + rd[j] = ((rd[j + 1] << 1) | 1) & char_match; + } else { + // Subsequent passes: fuzzy match. + rd[j] = ((rd[j + 1] << 1) | 1) & char_match | (((last_rd[j + 1] | last_rd[j]) << 1) | 1) | last_rd[j + 1]; + } + if ((rd[j] & matchmask) != 0) { + const score = matchBitapScore(d, j - 1, loc, pattern); + // This match will almost certainly be better than any existing + // match. But check anyway. + if (score <= threshold) { + // Told you so. + threshold = score; + best_loc = j - 1; + if (best_loc > loc) { + // When passing loc, don't exceed our current distance from loc. + start = @max(1, 2 * loc - best_loc); + } else { + // Already passed loc, downhill from here on in. + break; + } + } + } + } + if (matchBitapScore(d + 1, loc, loc, pattern) > threshold) { + // No hope for a (better) match at greater error levels. + break; + } + allocator.free(last_rd); + last_rd = rd; + } + allocator.free(last_rd); + return best_loc; +} + +/// Compute and return the score for a match with e errors and x location. +/// @param e Number of errors in match. +/// @param x Location of match. +/// @param loc Expected location of match. +/// @param pattern Pattern being sought. +/// @return Overall score for match (0.0 = good, 1.0 = bad). +fn matchBitapScore(e: usize, x: usize, loc: usize, pattern: []const u8) f64 { + // shortcut? TODO, proof in comments + // if (e == 0 and x == loc) return 0.0; + const e_float: f32 = @floatFromInt(e); + const len_float: f32 = @floatFromInt(pattern.len); + // if e == 0, accuracy == 0: 0/x = 0 + const accuracy = e_float / len_float; + // if loc == x, proximity == 0 + const proximity = if (loc >= x) loc - x else x - loc; + if (@This().match_distance == 0) { + // Dodge divide by zero + if (proximity == 0) // therefore this returns 0 + return accuracy + else + return 1.0; + } + const float_match: f64 = @floatFromInt(@This().match_distance); + // or this is 0 + 0/f_m aka 0 + return accuracy + (proximity / float_match); +} + +/// Initialise the alphabet for the Bitap algorithm. +/// @param pattern The text to encode. +/// @return Hash of character locations. +fn matchAlphabet(allocator: Allocator, pattern: []const u8) !std.HashMap(u8, usize) { + var map = std.HashMap(u8, usize).init(allocator); + errdefer map.deinit(); + for (pattern) |c| { + if (!map.contains(c)) { + try map.put(c, 0); + } + } + for (pattern, 0..) |c, i| { + const shift: u6 = @intCast(pattern.len - i - 1); + const value: usize = map.get(c) | (1 << shift); + try map.put(c, value); + } + return map; +} + +/// Initialise the alphabet for the Bitap algorithm. +/// @param pattern The text to encode. +/// @return Hash of character locations. +fn matchAlphabetImproved(allocator: Allocator, pattern: []const u8, UIntSize: type) !std.HashMap(u8, usize) { + const ShiftType = ShiftSizeForType(UIntSize); + var map = std.HashMap(u8, usize).init(allocator); + errdefer map.deinit(); + for (pattern) |c| { + if (!map.contains(c)) { + try map.put(c, 0); + } + } + for (pattern, 0..) |c, i| { + const shift: ShiftType = @intCast(pattern.len - i - 1); + // TODO I think we want c_mask & ~ 1 << shift here: + const value: UIntSize = map.get(c) | (1 << shift); + try map.put(c, value); + } + return map; +} + +//| PATCH FUNCTIONS + +/// +/// Increase the context until it is unique, but don't let the pattern +/// expand beyond DiffMatchPatch.match_max_bits. +/// +/// @param patch The patch to grow. +/// @param text Source text. +fn patchAddContext(allocator: Allocator, patch: *Patch, text: []const u8) !void { + if (text.len == 0) return; + // TODO the fixup logic here might make patterns too large? + // It should be ok, because big patches get broken up. Hmm. + var padding = 0; + { // Grow the pattern around the patch until unique, to set padding amount. + var pattern = text[patch.start2 .. patch.start2 + patch.length1]; + const max_width: usize = @This().match_max_bits - (2 * @This().patch_margin); + while (std.mem.indexOf(u8, text, pattern) != std.mem.lastIndexOf(u8, text, pattern) and pattern.len < max_width) { + padding += @This().patch_margin; + const pat_start = @max(0, patch.start2 - padding); + const pat_end = pat_start + @min(text.len, patch.start2 + patch.length1 + padding); + pattern = text[pat_start..pat_end]; + } + } + // Add one chunk for good luck. + padding += @This().patch_margin; + // Add the prefix. + const prefix = pre: { + var pre_start = @max(0, patch.start2 - padding); + // Make sure we're not breaking a codepoint. + while (is_follow(text[pre_start]) and pre_start > 0) { + pre_start -= 1; + } // Assuming we did everything else right, pre_end should be + // properly placed. + const pre_end = pre_start + patch.start2; + break :pre text[pre_start..pre_end]; + }; + if (prefix.len != 0) { + try patch.diffs.append( + allocator, + Diff{ + .operation = .equal, + .text = try allocator.dupe(u8, prefix), + }, + ); + } + // Add the suffix. + const suffix = post: { + const post_start = patch.start2 + patch.length1; + // In case we messed up somewhere: + assert(!is_follow(text[post_start])); + var post_end = post_start + @min(text.len, patch.start2 + patch.length1 + padding); + // Prevent broken codepoints here as well: Lead bytes, or follow with another follow + while (!std.ascii.isASCII(text[post_end]) and post_end + 1 < text.len and is_follow(text[post_end + 1])) { + post_end += 1; + // Special case: penultimate with another follow at end + if (post_end + 2 == text.len and is_follow(text[post_end + 1])) { + post_end += 1; + break; // Not actually necessary, but polite. + } + } + break :post text[post_start..post_end]; + }; + if (suffix.len != 0) { + try patch.diffs.append( + allocator, + Diff{ + .operation = .equal, + .text = try allocator.dupe(u8, suffix), + }, + ); + } + // Roll back the start points. + patch.start1 -= prefix.len; + patch.start2 -= prefix.len; + // Extend the lengths. + patch.length1 += prefix.len + suffix.len; + patch.length2 += prefix.len + suffix.len; +} + +/// Determines how to handle Diffs in a patch. Functions which create +/// the diffs internally can use `.own`: the Diffs will be copied to +/// the patch list, new ones allocated, and old ones freed. Then call +/// `deinit` on the DiffList, but not `deinitDiffList`. This *must not* +/// be used if the DiffList is not immediately freed, because some of +/// the diffs will contain spuriously empty text. +/// +/// Functions which operate on an existing DiffList should use `.copy`: +/// as the name indicates, copies of the Diffs will be made, and the +/// original memory must be freed separately. +const DiffHandling = enum { + copy, + own, +}; + +/// @return List of Patch objects. +fn makePatchInternal( + allocator: Allocator, + text: []const u8, + diffs: DiffList, + diff_act: DiffHandling, +) !PatchList { + const patches = PatchList{}; + if (diffs.items.len == 0) { + return patches; // Empty diff means empty patchlist + } + + var patch = Patch{}; + var char_count1 = 0; + var char_count2 = 0; + // This avoids freeing the original copy of the text: + var first_patch = true; + var prepatch_text = text; + defer { + if (!first_patch) + allocator.free(prepatch_text); + } + var postpatch = try std.ArrayList(u8).initCapacity(allocator, text.len); + defer postpatch.deinit(); + try postpatch.appendSlice(text); + for (diffs) |a_diff| { + if (patch.diffs.items.len == 0 and a_diff.operation != .equal) { + patch.start1 = char_count1; + patch.start2 = char_count2; + } + switch (a_diff.operation) { + .insert => { + const d = if (diff_act == .copy) a_diff.clone(allocator) else a_diff; + try patch.diffs.append(allocator, d); + patch.length2 += a_diff.text.len; + try postpatch.insertSlice(char_count2, a_diff.text); + }, + .delete => { + // + const d = if (diff_act == .copy) a_diff.clone(allocator) else a_diff; + try patch.diffs.append(allocator, d); + patch.length1 += a_diff.text.len; + try postpatch.replaceRange(char_count2, a_diff.text.len, .{}); + }, + .equal => { + // + if (a_diff.text.len <= 2 * @This().patch_margin and patch.diffs.items.len != 0 and a_diff != diffs.items[diffs.items.len]) { + // Small equality inside a patch. + const d = if (diff_act == .copy) a_diff.clone(allocator) else a_diff; + try patch.diffs.append(allocator, d); + patch.length1 += a_diff.text.len; + patch.length2 += a_diff.text.len; + } + if (a_diff.text.len >= 2 * @This().patch_margin) { + // Time for a new patch. + if (patch.diffs.items.len != 0) { + // free the Diff if we own it + if (diff_act == .own) { + allocator.free(a_diff.text); + } + try patchAddContext(allocator, patch, prepatch_text); + try patches.append(allocator, patch); + patch = Patch{}; + // Unlike Unidiff, our patch lists have a rolling context. + // https://github.com/google/diff-match-patch/wiki/Unidiff + // Update prepatch text & pos to reflect the application of the + // just completed patch. + if (first_patch) { + // no free on first + first_patch = false; + } else { + allocator.free(prepatch_text); + } + prepatch_text = try allocator.dupe(u8, postpatch.items); + char_count1 = char_count2; + } + } + }, + } + // Update the current character count. + if (a_diff.operation != .insert) { + char_count1 += a_diff.text.len; + } + if (a_diff.operation != .remove) { + char_count2 += a_diff.text.len; + } + } // end for loop + + // Pick up the leftover patch if not empty. + if (patch.diffs.items.len != 0) { + try patchAddContext(allocator, patch, prepatch_text); + try patches.append(allocator, patch); + } +} + +/// Compute a list of patches to turn text1 into text2. +/// text2 is not provided, diffs are the delta between text1 and text2. +/// +/// @param text1 Old text. +/// @param diffs Array of Diff objects for text1 to text2. +pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !PatchList { + try makePatchInternal(allocator, text, diffs, .copy); +} + +pub fn makePatchFromTexts(allocator: Allocator, text1: []const u8, text2: []const u8) !PatchList { + const diffs = try diff(@This(), allocator, text1, text2, true); + if (diffs.items.len > 2) { + try diffCleanupSemantic(diffs); + try diffCleanupEfficiency(diffs); + } + return try makePatchInternal(allocator, text1, diffs, .own); +} + +pub fn makePatchFromDiffs(allocator: Allocator, diffs: DiffList) !PatchList { + const text1 = try diffBeforeText(allocator, diffs); + return try makePatch(allocator, text1, diffs, .copy); +} + +/// Merge a set of patches onto the text. Returns a tuple: the first of which +/// is the patched text, the second of which is... +/// +/// TODO I'm just going to return a boolean saying whether all patches +/// were successful. Rethink this at some point. +/// +/// @param patches Array of Patch objects +/// @param text Old text. +/// @return Two element Object array, containing the new text and an array of +/// bool values. +pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const u8) !struct { []const u8, bool } { + if (og_patches.items.len == 0) { + // As silly as this is, we dupe the text, because something + // passing an empty patchset isn't going to check, and will + // end up double-freeing if we don't. Going with 'true' as + // the null patchset was successfully 'applied' here. + return .{ try allocator.dupe(u8, og_text), true }; + } + // So we can report if all patches were applied: + var all_applied = true; + // Deep copy the patches so that no changes are made to originals. + const patches = try patchListClone(allocator, og_patches); + defer patches.deinit(allocator); + const null_padding = try patchAddPadding(patches); + var text_array = try std.ArrayList(u8).initCapacity(og_text.len); + defer text_array.deinit(); + text_array.appendSlice(null_padding); + text_array.appendSlice(og_text); + text_array.appendSlice(null_padding); + try patchSplitMax(allocator, patches); + // delta keeps track of the offset between the expected and actual + // location of the previous patch. If there are patches expected at + // positions 10 and 20, but the first patch was found at 12, delta is 2 + // and the second patch has an effective expected position of 22. + var delta: usize = 0; + for (patches) |a_patch| { + const expected_loc = a_patch.start2 + delta; + const text1 = try diffBeforeText(allocator, a_patch.diffs); + defer allocator.free(text1); + var maybe_start: ?usize = null; + var maybe_end: ?usize = null; + const m_max_b = @This().match_max_bits; + if (text1.len > m_max_b) { + // patchSplitMax will only provide an oversized pattern + // in the case of a monster delete. + maybe_start = matchMain( + allocator, + text_array.items, + text1[0..m_max_b], + expected_loc, + ); + if (maybe_start) |start| { + const e_start = text1.len - m_max_b; + maybe_end = matchMain( + allocator, + text_array.items, + text1[e_start..], + e_start + expected_loc, + ); + // No match if a) no end_loc or b) the matches cross each other. + if (maybe_end) |end| { + if (start >= end) { + maybe_start = null; + } + } else { + maybe_start = null; + } + } + } else { + maybe_start = matchMain(allocator, og_text, text1, expected_loc); + } + if (maybe_start) |start| { + // Found a match. :) + delta = start - expected_loc; + // results[x] = true; + const text2 = t2: { + if (maybe_end) |end| { + break :t2 og_text[start..@min(end + m_max_b, og_text.len)]; + } else { + break :t2 og_text[start..@min(start + text1.len, og_text.len)]; + } + }; + if (std.mem.eql(u8, text1, text2)) { + // Perfect match, just shove the replacement text in. + const diff_text = try diffAfterText(allocator, a_patch.diffs); + defer allocator.free(diff_text); + try text_array.replaceRange(start, text1.len, diff_text); + } else { + // Imperfect match. Run a diff to get a framework of equivalent + // indices. + const diffs = try diff( + @This(), + allocator, + text1, + text2, + false, + ); + const t1_l_float: f64 = @floatFromInt(text1.len); + const bad_match = diffLevenshtein(diffs) / t1_l_float > @This().patch_delete_threshold; + if (text1.len > m_max_b and bad_match) { + // The end points match, but the content is unacceptably bad. + // results[x] = false; + all_applied = false; + } else { + diffCleanupSemanticLossless(allocator, diffs); + var index1: usize = 0; + for (diffs) |a_diff| { + if (a_diff.operation != .equal) { + const index2 = diffIndex(diffs, index1); + if (a_diff.operation == .insert) { + // Insertion + try text_array.insertSlice(start + index2, a_diff.text); + } else if (a_diff.operation == .delete) { + // Deletion + try text_array.replaceRange( + start + index2, + diffIndex(diffs, index1 + a_diff.text.len), + .{}, + ); + } + if (a_diff.operation != .delete) { + index1 += a_diff.text.len; + } + } + } + } + } + } else { + // No match found. :( + all_applied = false; + // Subtract the delta for this failed patch from subsequent patches. + delta -= a_patch.length2 - a_patch.length1; + } + } + // strip padding + try text_array.replaceRange(0, null_padding.len, .{}); + text_array.items.len -= null_padding.len; + return .{ text_array.toOwnedSlice(), all_applied }; +} + +// Look through the patches and break up any which are longer than the +// maximum limit of the match algorithm. +// Intended to be called only from within patchApply. +// @param patches List of Patch objects. +fn patchSplitMax(allocator: Allocator, patches: PatchList) !PatchList { + const patch_size = @This().match_max_bits; + const patch_margin = @This().patch_margin; + const max_patch_len = patch_size - patch_size - patch_margin; + // Mutating an array while iterating it? Sure, lets! + var x = 0; + while (x < patches.len) : (x += 1) { + if (patches[x].length1 <= patch_size) continue; + // We have a big ol' patch. + const bigpatch = patches.orderedRemove(x); + defer bigpatch.deinit(allocator); + // Prevent incrementing past the next patch: + x -= 1; + var start1 = bigpatch.start1; + var start2 = bigpatch.start2; + // start with an empty precontext so that we can deinit consistently + var precontext = try allocator.alloc(u8, 0); + while (bigpatch.diffs.items.len != 0) { + // Create one of several smaller patches. + var patch = Patch{}; + var empty = true; + patch.start1 = start1 - precontext.items.len; + patch.start2 = start2 - precontext.items.len; + if (precontext.len != 0) { + patch.length2 = precontext.length; + patch.length1 = patch.length2; + try patch.diffs.append( + allocator, + Diff{ + .operation = .equal, + .text = precontext.toOwnedSlice(), + }, + ); + } + while (bigpatch.diffs.count != 0 and patch.length1 < max_patch_len) { + const diff_type = bigpatch.diffs[0].operation; + const diff_text = bigpatch.diffs[0].text; + if (diff_type == .insert) { + // Insertions are harmless. + patch.length2 += diff_text.len; + start2 += diff_text.len; + // Move the patch (transfers ownership) + const diff1 = bigpatch.diffs.orderedRemove(0); + patch.diffs.append(diff1); + empty = false; + } else if (cond: { + // zig fmt simply will not line break if clauses :/ + const a = diff_type == .delete; + const b = patch.diffs.items.len == 1; + const c = patch.diffs[0].operation == .equal; + const d = diff_text.len > 2 * patch_size; + break :cond a and b and c and d; + }) { + // This is a large deletion. Let it pass in one chunk. + patch.length1 += diff_text.len; + start1 += diff_text.len; + empty = false; + // Transfer to patch: + const diff1 = bigpatch.diffs.orderedRemove(0); + try patch.diffs.append(allocator, diff1); + } else { + // Deletion or equality. Only take as much as we can stomach. + const text_end = @min(diff_text.len, patch_size - patch.length1 - patch_margin); + const new_diff_text = diff_text[0..text_end]; + patch.length += new_diff_text.len; + start1 += new_diff_text.len; + if (diff_type == .equal) { + patch.length2 += diff_text.len; + start2 += diff_text.len; + } else { + empty = false; + } + // Now check if we did anything. + if (new_diff_text.len == diff_text.len) { + // We can reuse the diff. + const diff1 = bigpatch.diffs.orderedRemove(0); + try patch.diffs.append(allocator, diff1); + } else { + // Free and dupe + const old_diff = bigpatch.diffs[0]; + defer old_diff.deinit(allocator); + bigpatch.diffs[0] = Diff{ + .operation = diff_type, + .text = try allocator.dupe(u8, new_diff_text), + }; + } + } + } + // Compute the head context for the next patch. + const context_len: isize = precontext.len - patch_margin; + allocator.free(precontext); + if (context_len > 0) { + const after_text = try diffAfterText(allocator, patch.diffs); + defer allocator.free(after_text); + precontext = try allocator.dupe(u8, after_text[context_len..]); + } else { + precontext = try allocator.alloc(u8, 0); + } + // Append the end context for this patch. + const post_text = try diffBeforeText(bigpatch.diffs); + const postcontext = post: { + if (post_text.len > patch_margin) { + defer allocator.free(post_text); + break :post post_text[0..patch_margin]; + } else { + break :post post_text; + } + }; + if (postcontext.len != 0) { + patch.length1 += postcontext.len; + patch.length2 += postcontext.len; + const maybe_last_diff = patch.diffs.getLastOrNull(); + if (maybe_last_diff) |last_diff| { + if (last_diff.operation == .equal) { + // free this diff and swap in a new one + defer last_diff.deinit(allocator); + patch.diffs.items.len -= 1; + const new_diff_text = try std.mem.concat( + allocator, + last_diff.text, + postcontext, + ); + try patch.diffs.append( + allocator, + Diff{ .operation = .equal, .text = new_diff_text }, + ); + } + } else { + // New diff from postcontext. + try patch.diffs.append( + allocator, + Diff{ .operation = .equal, .text = postcontext }, + ); + } + } else { + // We didn't allocate memory, but it's polite to free it (?) + allocator.free(postcontext); + } + if (!empty) { + // Insert the next patch + // Goes after x, and we need increment to skip: + x += 1; + try patches.insert(allocator, x, patch); + } + } + // Free final precontext. + allocator.free(precontext); + } +} + +/// Add some padding on text start and end so that edges can match something. +/// Intended to be called only from within patchApply. +/// @param patches Array of Patch objects. +/// @return The padding string added to each side. +fn patchAddPadding(allocator: Allocator, patches: PatchList) ![]const u8 { + assert(patches.items.len != 0); + const pad_len = @This().patch_margin; + var paddingcodes = try std.ArrayList(u8).initCapacity(allocator, pad_len); + defer paddingcodes.deinit(); + { + var control_code: u8 = 1; + while (control_code <= pad_len) : (control_code += 1) { + try paddingcodes.append(control_code); + } + } + // Bump all the patches forward. + for (patches) |a_patch| { + a_patch.start1 += pad_len; + a_patch.start2 += pad_len; + } + // Add some padding on start of first diff. + var patch = patches.items[0]; + var diffs = patch.diffs; + if (diffs.items.len == 0 or diffs.items[0].operation != .equal) { + // Add nullPadding equality. + try diffs.insert( + allocator, + 0, + Diff{ + .operation = .equal, + .text = try allocator.dupe(u8, paddingcodes.items), + }, + ); + patch.start1 -= pad_len; + // OG code says "Should be 0" but this statement is not justified... + assert(patch.start1 == 0); + patch.start2 -= pad_len; + assert(patch.start2 == 0); + patch.length1 += pad_len; + patch.lenght2 += pad_len; + } else if (pad_len > diffs.items[0].text.len) { + // Grow first equality. + var diff1 = diffs.items[0]; + defer allocator.free(diff1.text); + const extra_len = pad_len - diff1.text.len; + diff1.text = try std.mem.concat( + allocator, + paddingcodes.items[diff1.text.len..], + diff1.text, + ); + patch.start1 -= extra_len; + patch.start2 -= extra_len; + patch.length1 += extra_len; + patch.length2 += extra_len; + } + // Add some padding on end of last diff. + patch = patches.getLast(); + diffs = patch.diffs; + if (diffs.items.len == 0 or diffs.getLast().opeation != .equal) { + // Add nullPadding equality. + diffs.append( + allocator, + Diff{ + .operation = .equal, + .text = try allocator.dupe(u8, paddingcodes.items), + }, + ); + patch.length1 += pad_len; + patch.length2 += pad_len; + } else if (pad_len > diffs.getLast().text.len) { + // Grow last equality. + var last_diff = diffs.getLast(); + defer allocator.free(last_diff.text); + const extra_len = pad_len - last_diff.text.len; + last_diff.text = try std.mem.concat( + allocator, + last_diff.text, + paddingcodes[0..extra_len], + ); + patch.length1 += extra_len; + patch.length2 += extra_len; + } + return paddingcodes.toOwnedSlice(); +} + +/// Given an array of patches, return another array that is identical. +/// @param patches Array of Patch objects. +/// @return Array of Patch objects. +fn patchListClone(allocator: Allocator, patches: PatchList) !PatchList { + var new_patches = PatchList{}; + errdefer { + for (new_patches) |p| { + p.deinit(allocator); + } + } + new_patches.initCapacity(allocator, patches.items.len); + for (patches) |patch| { + try new_patches.append(allocator, try patch.clone(allocator)); + } + return new_patches; +} + +/// Take a list of patches and return a textual representation. +/// @param patches List of Patch objects. +/// @return Text representation of patches. +pub fn patchToText(allocator: Allocator, patches: PatchList) ![]const u8 { + const text_array = try std.ArrayList(u8).init(allocator); + defer text_array.deinit(); + const writer = text_array.writer(); + try writePatch(writer, patches); + return text_array.toOwnedSlice(); +} + +/// Stream a `PatchList` to the provided Writer. +pub fn writePatch(writer: anytype, patches: PatchList) !void { + for (patches) |a_patch| { + try a_patch.writePatch(writer); + } +} + +/// Parse a textual representation of patches and return a List of Patch +/// objects. +/// @param textline Text representation of patches. +/// @return List of Patch objects. +/// @throws ArgumentException If invalid input. +pub fn patchFromText(allocator: Allocator, text: []const u8) !PatchList { + if (text.len == 0) return PatchList{}; + var patches = PatchList{}; + var cursor = 0; + while (cursor < text.len) { + // TODO catch BadPatchString here and print diagnostic + const cursor_delta, const patch = try patchFromHeader(allocator, text[cursor..]); + cursor += cursor_delta; + try patches.append(allocator, patch); + } +} + +fn countDigits(text: []const u8) usize { + var idx = 0; + while (std.ascii.isDigit(text[idx])) : (idx += 1) {} + return idx; +} + +fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patch } { + var patch = Patch{}; + var cursor: usize = undefined; + if (std.mem.eql(u8, text[0..4], PATCH_HEAD)) { + // Parse location and length in before text + patch.start1 = std.fmt.parseInt( + usize, + text[4..], + 10, + ) catch return error.BadPatchString; + cursor = 4 + countDigits(text[4..]); + assert(cursor > 4); + if (text[cursor] != ',') { + cursor += 1; + patch.start1 -= 1; + patch.length1 = 1; + } else { + cursor += 1; + patch.length1 = std.fmt.parseInt( + usize, + text[cursor..], + 10, + ) catch return error.BadPatchString; + const delta = countDigits(text[cursor..]); + assert(delta > 0); + cursor += delta; + if (patch.length1 != 0) { + patch.start1 -= 1; + } + } + } else return error.BadPatchString; + // Parse location and length in after text. + if (text[cursor] == ' ' and text[cursor + 1] == '+') { + cursor += 2; + patch.start2 = std.fmt.parseInt( + usize, + text[cursor..], + 10, + ) catch return error.BadPatchString; + const delta1 = 4 + countDigits(text[4..]); + assert(delta1 > 0); + cursor += delta1; + if (text[cursor] != ',') { + cursor += 1; + patch.start2 -= 1; + patch.length2 = 1; + } else { + cursor += 1; + patch.length2 = std.fmt.parseInt( + usize, + text[cursor..], + 10, + ) catch return error.BadPatchString; + const delta2 = countDigits(text[cursor..]); + assert(delta2 > 1); + cursor += delta2; + if (patch.length2 != 0) { + patch.start2 -= 1; + } + } + } else return error.BadPatchString; + if (std.mem.eql(u8, text[cursor .. cursor + 4], PATCH_TAIL)) { + cursor += 4; + } else return error.BadPatchString; + // Eat the diffs + const patch_lines = std.mem.splitScalar( + u8, + text[cursor..], + '\n', + ); + // `splitScalar` means blank lines, but we need that to + // track the cursor. + patch_loop: while (patch_lines.next()) |line| { + cursor += line.len + 1; + if (line.len == 0) continue; + // Microsoft encodes spaces as +, we don't, so we don't need this: + // line = line.Replace("+", "%2b"); + const diff_line = try decodeUri(allocator, line) catch return error.BadPatchString; + switch (line[0]) { + '+' => { // Insertion + try patch.diffs.append( + allocator, + Diff{ + .operation = .insert, + .text = diff_line, + }, + ); + }, + '-' => { // Deletion + try patch.diffs.append( + allocator, + Diff{ + .operation = .delete, + .text = diff_line, + }, + ); + }, + ' ' => { // Minor equality + try patch.diffs.append( + allocator, + Diff{ + .operation = .equal, + .text = diff_line, + }, + ); + }, + '@' => { // Start of next patch + // back out cursor + cursor -= line.len - 1; + break :patch_loop; + }, + else => return error.BadPatchString, + } + } // end while + return .{ cursor, patch }; +} + +/// Decode our URI-esque escaping +fn decodeUri(allocator: Allocator, line: []const u8) ![]const u8 { + if (std.mem.indexOf(u8, line, '%')) |first| { + // Text to decode. + // Result will always be shorter than line: + var new_line = try std.ArrayList(u8).initCapacity(allocator, line.len); + defer new_line.init; + try new_line.appendSlice(line[0..first]); + var out_buf: [1]u8 = .{0}; + var codeunit = try std.fmt.hexToBytes(&out_buf, line[first + 1 .. first + 3]); + try new_line.append(codeunit[0]); + var cursor = first + 3; + while (std.mem.indexOf(u8, line[cursor..], '%')) |next| { + codeunit = try std.fmt.hexToBytes(&out_buf, line[next + 1 .. next + 3]); + try new_line.append(codeunit[0]); + cursor = next + 3; + } else { + try new_line.appendSlice(line[cursor..]); + } + return new_line.toOwnedSlice(); + } else { + return allocator.dupe(u8, line); + } +} + +/// +/// Borrowed from https://github.com/elerch/aws-sdk-for-zig/blob/master/src/aws_http.zig +/// under the MIT license. Thanks! +/// +/// Modified to implement Unidiff escaping, documented here: +/// https://github.com/google/diff-match-patch/wiki/Unidiff +/// +/// The documentation reads: +/// +/// > Special characters are encoded using %xx notation. The set of +/// > characters which are encoded matches JavaScript's `encodeURI()` +/// > function, with the exception of spaces which are not encoded. +/// +/// So we encode everything but the characters defined by Moz: +/// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURI +/// +/// These: !#$&'()*+,-./:;=?@_~ (and alphanumeric ASCII) +/// +/// There is a nice contiguous run of 10 symbols between `&` and `/`, which we +/// can test in two comparisons, leaving these assorted: +/// +/// !#$:;=?@_~ +/// +/// Each URI encoded byte is formed by a '%' and the two-digit +/// hexadecimal value of the byte. +/// +/// Letters in the hexadecimal value must be uppercase, for example "%1A". +/// +fn writeUriEncoded(writer: anytype, text: []const u8) !usize { + const remaining_characters = "!#$:;=?@_~"; + var written: usize = 0; + for (text) |c| { + const should_encode = should: { + if (c == ' ' or std.ascii.isAlphanumeric(c)) { + break :should false; + } + if ('&' <= c and c <= '/') { + break :should false; + } + for (remaining_characters) |r| { + if (r == c) { + break :should false; + } + } + break :should true; + }; + + if (!should_encode) { + try writer.writeByte(c); + written += 1; + continue; + } + // Whatever remains, encode it + try writer.writeByte('%'); + written += 1; + const hexen = std.fmt.bytesToHex(&[_]u8{c}, .upper); + written += try writer.write(&hexen); + } + return written; +} + +fn encodeUri(allocator: std.mem.Allocator, text: []const u8) ![]u8 { + var charlist = try std.ArrayList(u8).initCapacity(allocator, text.len); + defer charlist.deinit(); + const writer = charlist.writer(); + _ = try writeUriEncoded(writer, text); + return charlist.toOwnedSlice(); +} + +test encodeUri { + const allocator = std.testing.allocator; + const special_chars = "!#$&'()*+,-./:;=?@_~"; + const special_encoded = try encodeUri(allocator, special_chars); + defer allocator.free(special_encoded); + try testing.expectEqualStrings(special_chars, special_encoded); + const alphaspace = " ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + const alpha_encoded = try encodeUri(allocator, alphaspace); + defer allocator.free(alpha_encoded); + try testing.expectEqualStrings(alphaspace, alpha_encoded); + const to_encode = "\"%<>[\\]^`{|}δ"; + const encodes = try encodeUri(allocator, to_encode); + defer allocator.free(encodes); + try testing.expectEqualStrings("%22%25%3C%3E%5B%5C%5D%5E%60%7B%7C%7D%CE%B4", encodes); } // DONE [✅]: Allocate all text in diffs to -// not cause segfault while freeing +// not cause segfault while freeing; not a problem +// at the moment because we don't free anything :( +// (or was it??) test diffCommonPrefix { // Detect any common suffix. From dec0868097d4a562cae8f02c3d05833e15ebf7b1 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 00:07:38 -0400 Subject: [PATCH 074/176] diffIndex tests This was a hideous process I dearly hope to never repeat. I've never had a repo go this sour on me. Never again, VSCode. Never. Again. --- DiffMatchPatch.zig | 51 +++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 5241819..74b9e29 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1802,25 +1802,22 @@ fn diffCommonOverlap(text1_in: []const u8, text2_in: []const u8) usize { /// @param loc Location within text1. /// @return Location within text2. /// -pub fn diffIndex(diffs: DiffList, loc: usize) usize { - // int chars1 = 0; - // int chars2 = 0; - // int last_chars1 = 0; - // int last_chars2 = 0; - var chars1: usize = 0; - var chars2: usize = 0; - var last_chars1: usize = 0; - var last_chars2: usize = 0; +pub fn diffIndex(diffs: DiffList, u_loc: usize) usize { + var chars1: isize = 0; + var chars2: isize = 0; + var last_chars1: isize = 0; + var last_chars2: isize = 0; + const loc: isize = @intCast(u_loc); // Dummy diff var last_diff: Diff = Diff{ .operation = .equal, .text = "" }; - for (diffs) |a_diff| { + for (diffs.items) |a_diff| { if (a_diff.operation != .insert) { // Equality or deletion. - chars1 += a_diff.text.len; + chars1 += @intCast(a_diff.text.len); } if (a_diff.operation != .delete) { // Equality or insertion. - chars2 += a_diff.text.len; + chars2 += @intCast(a_diff.text.len); } if (chars1 > loc) { // Overshot the location. @@ -1833,10 +1830,36 @@ pub fn diffIndex(diffs: DiffList, loc: usize) usize { if (last_diff.text.len != 0 and last_diff.operation == .delete) { // The location was deleted. - return last_chars2; + return @intCast(last_chars2); } // Add the remaining character length. - return last_chars2 + (loc - last_chars1); + return @intCast(last_chars2 + (loc - last_chars1)); +} + +test diffIndex { + const dmp = DiffMatchPatch{}; + { + var diffs = try dmp.diff( + testing.allocator, + "The midnight train", + "The blue midnight train", + false, + ); + defer deinitDiffList(testing.allocator, &diffs); + try testing.expectEqual(0, diffIndex(diffs, 0)); + try testing.expectEqual(9, diffIndex(diffs, 4)); + } + { + var diffs = try dmp.diff( + testing.allocator, + "Better still to live and learn", + "Better yet to learn and live", + false, + ); + defer deinitDiffList(testing.allocator, &diffs); + try testing.expectEqual(11, diffIndex(diffs, 13)); + try testing.expectEqual(20, diffIndex(diffs, 21)); + } } /// A struct holding bookends for `diffPrittyFormat(diffs)`. From 4bf0e1ca1a2cfdbf61eca5387d959abfd8303b9f Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 01:31:24 -0400 Subject: [PATCH 075/176] Add DiffMatchPatch to a bunch of things Also an eyeball run on head off allocation and other memory problems at the pass. --- DiffMatchPatch.zig | 204 ++++++++++++++++++++++++++++----------------- 1 file changed, 126 insertions(+), 78 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 74b9e29..4d59b40 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1895,13 +1895,18 @@ pub fn diffPrettyFormat( diffs: DiffList, deco: DiffDecorations, ) ![]const u8 { - var out = ArrayListUnmanaged(u8){}; - defer out.deinit(allocator); + var out = std.ArrayList(u8).init(allocator); + defer out.deinit(); const writer = out.writer(); _ = try writeDiffPrettyFormat(allocator, writer, diffs, deco); return out.toOwnedSlice(allocator); } +/// Pretty-print a diff for output to a terminal. +pub fn diffPrettyFormatXTerm(allocator: Allocator, diffs: DiffList) ![]const u8 { + return try diffPrettyFormat(allocator, diffs, xterm_classic); +} + /// Write a pretty-formatted `DiffList` to `writer`. The `Allocator` /// is only used if a custom text formatter is defined for /// `DiffDecorations`. Returns number of bytes written. @@ -1951,7 +1956,7 @@ pub fn writeDiffPrettyFormat( pub fn diffBeforeText(allocator: Allocator, diffs: DiffList) ![]const u8 { var chars = ArrayListUnmanaged(u8){}; defer chars.deinit(allocator); - for (diffs) |d| { + for (diffs.items) |d| { if (d.operation != .insert) { try chars.appendSlice(allocator, d.text); } @@ -1967,7 +1972,7 @@ pub fn diffBeforeText(allocator: Allocator, diffs: DiffList) ![]const u8 { pub fn diffAfterText(allocator: Allocator, diffs: DiffList) ![]const u8 { var chars = ArrayListUnmanaged(u8){}; defer chars.deinit(allocator); - for (diffs) |d| { + for (diffs.items) |d| { if (d.operation != .delete) { try chars.appendSlice(allocator, d.text); } @@ -1986,7 +1991,7 @@ pub fn diffLevenshtein(diffs: DiffList) usize { var inserts: usize = 0; var deletes: usize = 0; var levenshtein: usize = 0; - for (diffs) |d| { + for (diffs.items) |d| { switch (d.operation) { .insert => { inserts += d.text.len; @@ -2014,7 +2019,13 @@ pub fn diffLevenshtein(diffs: DiffList) usize { /// @param pattern The pattern to search for. /// @param loc The location to search around. /// @return Best match index or -1. -pub fn matchMain(allocator: Allocator, text: []const u8, pattern: []const u8, passed_loc: usize) ?usize { +pub fn matchMain( + dmp: DiffMatchPatch, + allocator: Allocator, + text: []const u8, + pattern: []const u8, + passed_loc: usize, +) ?usize { // Clamp the loc to fit within text. const loc = @min(passed_loc, text.len); if (std.mem.eql(u8, text, pattern)) { @@ -2029,7 +2040,7 @@ pub fn matchMain(allocator: Allocator, text: []const u8, pattern: []const u8, pa return loc; } else { // Do a fuzzy compare. - return matchBitap(allocator, text, pattern, loc); + return dmp.matchBitap(allocator, text, pattern, loc); } } @@ -2041,6 +2052,7 @@ pub fn matchMain(allocator: Allocator, text: []const u8, pattern: []const u8, pa // a nice balance between code size and versatility. // Something like this: fn matchBitapImproved( + dmp: DiffMatchPatch, allocator: Allocator, text: []const u8, pattern: []const u8, @@ -2053,17 +2065,17 @@ fn matchBitapImproved( var map = try matchAlphabet(allocator, pattern); defer map.deinit(); // Highest score beyond which we give up. - var threshold = @This().threshold; + var threshold = dmp.threshold; // Is there a nearby exact match? (speedup) var best_loc = std.mem.indexOfPos(u8, text, pattern); if (best_loc) |best| { - threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), threshold); } // What about in the other direction? (speedup) const trunc_text = text[0..@min(loc + pattern.len, text.len)]; best_loc = std.mem.lastIndexOf(u8, trunc_text, pattern); if (best_loc) |best| { - threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), threshold); } // Initialise the bit arrays. const shift: ShiftWidth = @intCast(pattern.len - 1); @@ -2082,7 +2094,7 @@ fn matchBitapImproved( bin_min = 0; bin_mid = bin_max; while (bin_min < bin_mid) { - if (matchBitapScore(d, loc + bin_mid, loc, pattern) <= threshold) { + if (dmp.matchBitapScore(d, loc + bin_mid, loc, pattern) <= threshold) { bin_min = bin_mid; } else { bin_max = bin_mid; @@ -2111,7 +2123,7 @@ fn matchBitapImproved( rd[j] = ((rd[j + 1] << 1)) & char_match & (((last_rd[j + 1] & last_rd[j]) << 1)) & last_rd[j + 1]; } if ((rd[j] & matchmask) != 0) { - const score = matchBitapScore(d, j - 1, loc, pattern); + const score = dmp.matchBitapScore(d, j - 1, loc, pattern); // This match will almost certainly be better than any existing // match. But check anyway. if (score <= threshold) { @@ -2128,7 +2140,7 @@ fn matchBitapImproved( } } } - if (matchBitapScore(d + 1, loc, loc, pattern) > threshold) { + if (dmp.matchBitapScore(d + 1, loc, loc, pattern) > threshold) { // No hope for a (better) match at greater error levels. break; } @@ -2157,6 +2169,7 @@ fn ShiftSizeForType(T: type) type { /// @param loc The location to search around. /// @return Best match index or -1. fn matchBitap( + dmp: DiffMatchPatch, allocator: Allocator, text: []const u8, pattern: []const u8, @@ -2170,11 +2183,11 @@ fn matchBitap( var map = try matchAlphabet(allocator, pattern); defer map.deinit(); // Highest score beyond which we give up. - var threshold = @This().threshold; + var threshold = dmp.threshold; // Is there a nearby exact match? (speedup) var best_loc = std.mem.indexOfPos(u8, text, pattern); if (best_loc) |best| { - threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), threshold); } // TODO obviously if we want a speedup here, we do this: // if (threshold == 0.0) return best_loc; @@ -2193,7 +2206,8 @@ fn matchBitap( var bin_mid: usize = undefined; var bin_max = pattern.len + text.len; // null last_rd to simplying freeing memory - var last_rd: []usize = try allocator.alloct(usize, 0); + var last_rd: []usize = try allocator.alloc(usize, 0); + errdefer allocator.free(last_rd); for (0..pattern.len) |d| { // Scan for the best match; each iteration allows for one more error. // Run a binary search to determine how far from 'loc' we can stray at @@ -2212,7 +2226,8 @@ fn matchBitap( bin_max = bin_mid; var start = @max(1, loc - bin_mid + 1); const finish = @min(loc + bin_mid, text.len) + pattern.len; - var rd: []usize = allocator.alloc(usize, finish + 2); + // No errors below this point, so no errdefer either: + var rd = try allocator.alloc(usize, finish + 2); const dshift: u6 = @intCast(d); rd[finish + 1] = (1 << dshift) - 1; var j = finish; @@ -2327,29 +2342,34 @@ fn matchAlphabetImproved(allocator: Allocator, pattern: []const u8, UIntSize: ty //| PATCH FUNCTIONS -/// /// Increase the context until it is unique, but don't let the pattern /// expand beyond DiffMatchPatch.match_max_bits. /// /// @param patch The patch to grow. /// @param text Source text. -fn patchAddContext(allocator: Allocator, patch: *Patch, text: []const u8) !void { +fn patchAddContext( // XXX pick it up from here + dmp: DiffMatchPatch, + allocator: Allocator, + patch: *Patch, + text: []const u8, +) !void { if (text.len == 0) return; // TODO the fixup logic here might make patterns too large? // It should be ok, because big patches get broken up. Hmm. + // Also, the SimpleNote maintained branch does it this way. var padding = 0; { // Grow the pattern around the patch until unique, to set padding amount. var pattern = text[patch.start2 .. patch.start2 + patch.length1]; - const max_width: usize = @This().match_max_bits - (2 * @This().patch_margin); + const max_width: usize = dmp.match_max_bits - (2 * dmp.patch_margin); while (std.mem.indexOf(u8, text, pattern) != std.mem.lastIndexOf(u8, text, pattern) and pattern.len < max_width) { - padding += @This().patch_margin; + padding += dmp.patch_margin; const pat_start = @max(0, patch.start2 - padding); const pat_end = pat_start + @min(text.len, patch.start2 + patch.length1 + padding); pattern = text[pat_start..pat_end]; } } // Add one chunk for good luck. - padding += @This().patch_margin; + padding += dmp.patch_margin; // Add the prefix. const prefix = pre: { var pre_start = @max(0, patch.start2 - padding); @@ -2362,12 +2382,12 @@ fn patchAddContext(allocator: Allocator, patch: *Patch, text: []const u8) !void break :pre text[pre_start..pre_end]; }; if (prefix.len != 0) { - try patch.diffs.append( - allocator, - Diff{ - .operation = .equal, - .text = try allocator.dupe(u8, prefix), - }, + try patch.diffs.ensureUnusedCapacity(allocator, 1); + patch.diffs.appendAssumeCapacity( + Diff.init( + .equal, + try allocator.dupe(u8, prefix), + ), ); } // Add the suffix. @@ -2388,12 +2408,12 @@ fn patchAddContext(allocator: Allocator, patch: *Patch, text: []const u8) !void break :post text[post_start..post_end]; }; if (suffix.len != 0) { - try patch.diffs.append( - allocator, - Diff{ - .operation = .equal, - .text = try allocator.dupe(u8, suffix), - }, + try patch.diffs.ensureUnusedCapacity(allocator, 1); + patch.diffs.appendAssumeCapacity( + Diff.init( + .equal, + try allocator.dupe(u8, suffix), + ), ); } // Roll back the start points. @@ -2421,6 +2441,7 @@ const DiffHandling = enum { /// @return List of Patch objects. fn makePatchInternal( + dmp: DiffMatchPatch, allocator: Allocator, text: []const u8, diffs: DiffList, @@ -2444,7 +2465,7 @@ fn makePatchInternal( var postpatch = try std.ArrayList(u8).initCapacity(allocator, text.len); defer postpatch.deinit(); try postpatch.appendSlice(text); - for (diffs) |a_diff| { + for (diffs.items) |a_diff| { if (patch.diffs.items.len == 0 and a_diff.operation != .equal) { patch.start1 = char_count1; patch.start2 = char_count2; @@ -2465,19 +2486,21 @@ fn makePatchInternal( }, .equal => { // - if (a_diff.text.len <= 2 * @This().patch_margin and patch.diffs.items.len != 0 and a_diff != diffs.items[diffs.items.len]) { + if (a_diff.text.len <= 2 * dmp.patch_margin and patch.diffs.items.len != 0 and a_diff != diffs.items[diffs.items.len]) { // Small equality inside a patch. - const d = if (diff_act == .copy) a_diff.clone(allocator) else a_diff; - try patch.diffs.append(allocator, d); + patch.diffs.ensureUnusedCapacity(allocator, 1); + const d = if (diff_act == .copy) try a_diff.clone(allocator) else a_diff; + patch.diffs.appendAssumeCapacity(allocator, d); patch.length1 += a_diff.text.len; patch.length2 += a_diff.text.len; } - if (a_diff.text.len >= 2 * @This().patch_margin) { + if (a_diff.text.len >= 2 * dmp.patch_margin) { // Time for a new patch. if (patch.diffs.items.len != 0) { // free the Diff if we own it if (diff_act == .own) { allocator.free(a_diff.text); + a_diff.text = ""; // for errdefer } try patchAddContext(allocator, patch, prepatch_text); try patches.append(allocator, patch); @@ -2519,12 +2542,18 @@ fn makePatchInternal( /// /// @param text1 Old text. /// @param diffs Array of Diff objects for text1 to text2. -pub fn makePatch(allocator: Allocator, text: []const u8, diffs: DiffList) !PatchList { - try makePatchInternal(allocator, text, diffs, .copy); +pub fn makePatch( + dmp: DiffMatchPatch, + allocator: Allocator, + text: []const u8, + diffs: DiffList, +) !PatchList { + try dmp.makePatchInternal(allocator, text, diffs, .copy); } pub fn makePatchFromTexts(allocator: Allocator, text1: []const u8, text2: []const u8) !PatchList { const diffs = try diff(@This(), allocator, text1, text2, true); + errdefer diffs.deinit(allocator); if (diffs.items.len > 2) { try diffCleanupSemantic(diffs); try diffCleanupEfficiency(diffs); @@ -2534,6 +2563,7 @@ pub fn makePatchFromTexts(allocator: Allocator, text1: []const u8, text2: []cons pub fn makePatchFromDiffs(allocator: Allocator, diffs: DiffList) !PatchList { const text1 = try diffBeforeText(allocator, diffs); + defer allocator.free(text1); return try makePatch(allocator, text1, diffs, .copy); } @@ -2547,7 +2577,12 @@ pub fn makePatchFromDiffs(allocator: Allocator, diffs: DiffList) !PatchList { /// @param text Old text. /// @return Two element Object array, containing the new text and an array of /// bool values. -pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const u8) !struct { []const u8, bool } { +pub fn patchApply( + dmp: DiffMatchPatch, + allocator: Allocator, + og_patches: PatchList, + og_text: []const u8, +) !struct { []const u8, bool } { if (og_patches.items.len == 0) { // As silly as this is, we dupe the text, because something // passing an empty patchset isn't going to check, and will @@ -2560,12 +2595,12 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const // Deep copy the patches so that no changes are made to originals. const patches = try patchListClone(allocator, og_patches); defer patches.deinit(allocator); - const null_padding = try patchAddPadding(patches); + const null_padding = try patchAddPadding(allocator, patches); var text_array = try std.ArrayList(u8).initCapacity(og_text.len); defer text_array.deinit(); - text_array.appendSlice(null_padding); - text_array.appendSlice(og_text); - text_array.appendSlice(null_padding); + try text_array.appendSlice(null_padding); + try text_array.appendSlice(og_text); + try text_array.appendSlice(null_padding); try patchSplitMax(allocator, patches); // delta keeps track of the offset between the expected and actual // location of the previous patch. If there are patches expected at @@ -2578,11 +2613,11 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const defer allocator.free(text1); var maybe_start: ?usize = null; var maybe_end: ?usize = null; - const m_max_b = @This().match_max_bits; + const m_max_b = dmp.match_max_bits; if (text1.len > m_max_b) { // patchSplitMax will only provide an oversized pattern // in the case of a monster delete. - maybe_start = matchMain( + maybe_start = dmp.matchMain( allocator, text_array.items, text1[0..m_max_b], @@ -2590,7 +2625,7 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const ); if (maybe_start) |start| { const e_start = text1.len - m_max_b; - maybe_end = matchMain( + maybe_end = dmp.matchMain( allocator, text_array.items, text1[e_start..], @@ -2606,12 +2641,12 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const } } } else { - maybe_start = matchMain(allocator, og_text, text1, expected_loc); + maybe_start = dmp.matchMain(allocator, og_text, text1, expected_loc); } if (maybe_start) |start| { // Found a match. :) delta = start - expected_loc; - // results[x] = true; + // results[x] = true; const text2 = t2: { if (maybe_end) |end| { break :t2 og_text[start..@min(end + m_max_b, og_text.len)]; @@ -2627,8 +2662,7 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const } else { // Imperfect match. Run a diff to get a framework of equivalent // indices. - const diffs = try diff( - @This(), + const diffs = try dmp.diff( allocator, text1, text2, @@ -2638,7 +2672,7 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const const bad_match = diffLevenshtein(diffs) / t1_l_float > @This().patch_delete_threshold; if (text1.len > m_max_b and bad_match) { // The end points match, but the content is unacceptably bad. - // results[x] = false; + // results[x] = false; all_applied = false; } else { diffCleanupSemanticLossless(allocator, diffs); @@ -2681,9 +2715,13 @@ pub fn patchApply(allocator: Allocator, og_patches: PatchList, og_text: []const // maximum limit of the match algorithm. // Intended to be called only from within patchApply. // @param patches List of Patch objects. -fn patchSplitMax(allocator: Allocator, patches: PatchList) !PatchList { - const patch_size = @This().match_max_bits; - const patch_margin = @This().patch_margin; +fn patchSplitMax( + dmp: DiffMatchPatch, + allocator: Allocator, + patches: PatchList, +) !PatchList { + const patch_size = dmp.match_max_bits; + const patch_margin = dmp.patch_margin; const max_patch_len = patch_size - patch_size - patch_margin; // Mutating an array while iterating it? Sure, lets! var x = 0; @@ -2707,11 +2745,12 @@ fn patchSplitMax(allocator: Allocator, patches: PatchList) !PatchList { if (precontext.len != 0) { patch.length2 = precontext.length; patch.length1 = patch.length2; - try patch.diffs.append( + try patch.diffs.ensureUnusedCapacity(allocator, 1); + patch.diffs.appendAssumeCapacity( allocator, Diff{ .operation = .equal, - .text = precontext.toOwnedSlice(), + .text = try precontext.toOwnedSlice(), }, ); } @@ -2739,8 +2778,9 @@ fn patchSplitMax(allocator: Allocator, patches: PatchList) !PatchList { start1 += diff_text.len; empty = false; // Transfer to patch: + try patch.diffs.ensureUnusedCapacity(allocator, 1); const diff1 = bigpatch.diffs.orderedRemove(0); - try patch.diffs.append(allocator, diff1); + patch.diffs.appendAssumeCapacity(diff1); } else { // Deletion or equality. Only take as much as we can stomach. const text_end = @min(diff_text.len, patch_size - patch.length1 - patch_margin); @@ -2756,16 +2796,18 @@ fn patchSplitMax(allocator: Allocator, patches: PatchList) !PatchList { // Now check if we did anything. if (new_diff_text.len == diff_text.len) { // We can reuse the diff. + try patch.diffs.ensureUnusedCapacity(allocator, 1); const diff1 = bigpatch.diffs.orderedRemove(0); - try patch.diffs.append(allocator, diff1); + patch.diffs.append(diff1); } else { // Free and dupe const old_diff = bigpatch.diffs[0]; - defer old_diff.deinit(allocator); + errdefer old_diff.deinit(allocator); bigpatch.diffs[0] = Diff{ .operation = diff_type, .text = try allocator.dupe(u8, new_diff_text), }; + old_diff.deinit(allocator); } } } @@ -2815,9 +2857,6 @@ fn patchSplitMax(allocator: Allocator, patches: PatchList) !PatchList { Diff{ .operation = .equal, .text = postcontext }, ); } - } else { - // We didn't allocate memory, but it's polite to free it (?) - allocator.free(postcontext); } if (!empty) { // Insert the next patch @@ -2835,9 +2874,13 @@ fn patchSplitMax(allocator: Allocator, patches: PatchList) !PatchList { /// Intended to be called only from within patchApply. /// @param patches Array of Patch objects. /// @return The padding string added to each side. -fn patchAddPadding(allocator: Allocator, patches: PatchList) ![]const u8 { +fn patchAddPadding( + dmp: DiffMatchPatch, + allocator: Allocator, + patches: PatchList, +) ![]const u8 { assert(patches.items.len != 0); - const pad_len = @This().patch_margin; + const pad_len = dmp.patch_margin; var paddingcodes = try std.ArrayList(u8).initCapacity(allocator, pad_len); defer paddingcodes.deinit(); { @@ -2856,8 +2899,8 @@ fn patchAddPadding(allocator: Allocator, patches: PatchList) ![]const u8 { var diffs = patch.diffs; if (diffs.items.len == 0 or diffs.items[0].operation != .equal) { // Add nullPadding equality. - try diffs.insert( - allocator, + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.insert( 0, Diff{ .operation = .equal, @@ -2891,8 +2934,8 @@ fn patchAddPadding(allocator: Allocator, patches: PatchList) ![]const u8 { diffs = patch.diffs; if (diffs.items.len == 0 or diffs.getLast().opeation != .equal) { // Add nullPadding equality. - diffs.append( - allocator, + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.appendAssumeCapacity( Diff{ .operation = .equal, .text = try allocator.dupe(u8, paddingcodes.items), @@ -2959,6 +3002,7 @@ pub fn writePatch(writer: anytype, patches: PatchList) !void { pub fn patchFromText(allocator: Allocator, text: []const u8) !PatchList { if (text.len == 0) return PatchList{}; var patches = PatchList{}; + errdefer patches.deinit(allocator); var cursor = 0; while (cursor < text.len) { // TODO catch BadPatchString here and print diagnostic @@ -2966,6 +3010,7 @@ pub fn patchFromText(allocator: Allocator, text: []const u8) !PatchList { cursor += cursor_delta; try patches.append(allocator, patch); } + return patches; } fn countDigits(text: []const u8) usize { @@ -2976,6 +3021,7 @@ fn countDigits(text: []const u8) usize { fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patch } { var patch = Patch{}; + errdefer patch.deinit(allocator); var cursor: usize = undefined; if (std.mem.eql(u8, text[0..4], PATCH_HEAD)) { // Parse location and length in before text @@ -3052,6 +3098,7 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc // Microsoft encodes spaces as +, we don't, so we don't need this: // line = line.Replace("+", "%2b"); const diff_line = try decodeUri(allocator, line) catch return error.BadPatchString; + errdefer allocator.free(diff_line); switch (line[0]) { '+' => { // Insertion try patch.diffs.append( @@ -3100,11 +3147,17 @@ fn decodeUri(allocator: Allocator, line: []const u8) ![]const u8 { defer new_line.init; try new_line.appendSlice(line[0..first]); var out_buf: [1]u8 = .{0}; - var codeunit = try std.fmt.hexToBytes(&out_buf, line[first + 1 .. first + 3]); + var codeunit = std.fmt.hexToBytes( + &out_buf, + line[first + 1 .. first + 3], + ) catch return error.BadPatchString; try new_line.append(codeunit[0]); var cursor = first + 3; while (std.mem.indexOf(u8, line[cursor..], '%')) |next| { - codeunit = try std.fmt.hexToBytes(&out_buf, line[next + 1 .. next + 3]); + codeunit = try std.fmt.hexToBytes( + &out_buf, + line[next + 1 .. next + 3], + ) catch return error.BadPatchString; try new_line.append(codeunit[0]); cursor = next + 3; } else { @@ -3201,11 +3254,6 @@ test encodeUri { try testing.expectEqualStrings("%22%25%3C%3E%5B%5C%5D%5E%60%7B%7C%7D%CE%B4", encodes); } -// DONE [✅]: Allocate all text in diffs to -// not cause segfault while freeing; not a problem -// at the moment because we don't free anything :( -// (or was it??) - test diffCommonPrefix { // Detect any common suffix. try testing.expectEqual(@as(usize, 0), diffCommonPrefix("abc", "xyz")); // Null case From 9e6a7e057eab6d2b237cb8070adac8549286c657 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 10:35:08 -0400 Subject: [PATCH 076/176] Tests for diffBefore and -AfterText --- DiffMatchPatch.zig | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 4d59b40..40e3569 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1980,6 +1980,21 @@ pub fn diffAfterText(allocator: Allocator, diffs: DiffList) ![]const u8 { return chars.toOwnedSlice(allocator); } +test "diff before and after text" { + const dmp = DiffMatchPatch{}; + const allocator = testing.allocator; + const before = "The cat in the hat."; + const after = "The bat in the belfry."; + var diffs = try dmp.diff(allocator, before, after, false); + defer deinitDiffList(allocator, &diffs); + const before1 = try diffBeforeText(allocator, diffs); + defer allocator.free(before1); + const after1 = try diffAfterText(allocator, diffs); + defer allocator.free(after1); + try testing.expectEqualStrings(before, before1); + try testing.expectEqualStrings(after, after1); +} + /// /// Compute the Levenshtein distance; the number of inserted, /// deleted or substituted characters. From 6ee94e52fa087bb56c43e4f5741a34f5be20709b Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 10:49:51 -0400 Subject: [PATCH 077/176] Enhancements --- roadmap.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/roadmap.md b/roadmap.md index 7c31c46..9a64ef0 100644 --- a/roadmap.md +++ b/roadmap.md @@ -4,15 +4,20 @@ - [ ] Add DiffMatchPatch object instead of @This() (which won't work) - [✅] Port match - [ ] Port test coverage -- [ ] Diff stream - - [ ] Use Unicode characters and codepoint indices - 32 - - [ ] Implement line diff as a stream - - [ ] Also gives word diff, token diff, etc. - [ ] Refactor: - [ ] Diff struct becomes Edit - [ ] DiffList stays - [ ] New Diff struct, and DiffUnmanaged - [ ] Namespaces subsequent operations on diffs +- [ ] Enhancements + - [ ] diffsForRegion: provides every diff pertaining to a specific + region of `before`. Needs to also include how much overlap, if + any, the diff includes. Should have "borrow" and "copy" + versions. + - [ ] Diff stream + - [ ] Use Unicode characters and codepoint indices - 32 + - [ ] Implement line diff as a stream + - [ ] Also gives word diff, token diff, etc. - [ ] Histogram? - [ ] Imara diff has an optimized histogram: https://github.com/pascalkuthe/imara-diff From d4fc092b4d98ea1c803810d7c6c582356734be40 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 10:49:56 -0400 Subject: [PATCH 078/176] Tests for diffLevenshtein --- DiffMatchPatch.zig | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 40e3569..e9f5399 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2006,13 +2006,13 @@ pub fn diffLevenshtein(diffs: DiffList) usize { var inserts: usize = 0; var deletes: usize = 0; var levenshtein: usize = 0; - for (diffs.items) |d| { - switch (d.operation) { + for (diffs.items) |a_diff| { + switch (a_diff.operation) { .insert => { - inserts += d.text.len; + inserts += a_diff.text.len; }, .delete => { - deletes += d.text.len; + deletes += a_diff.text.len; }, .equal => { // A deletion and an insertion is one substitution. @@ -2026,6 +2026,41 @@ pub fn diffLevenshtein(diffs: DiffList) usize { return levenshtein + @max(inserts, deletes); } +test diffLevenshtein { + const allocator = testing.allocator; + // These diffs don't get text freed + { + var diffs = DiffList{}; + defer diffs.deinit(allocator); + try diffs.appendSlice(allocator, &.{ + Diff.init(.delete, "abc"), + Diff.init(.insert, "1234"), + Diff.init(.equal, "xyz"), + }); + try testing.expectEqual(4, diffLevenshtein(diffs)); + } + { + var diffs = DiffList{}; + defer diffs.deinit(allocator); + try diffs.appendSlice(allocator, &.{ + Diff.init(.equal, "xyz"), + Diff.init(.delete, "abc"), + Diff.init(.insert, "1234"), + }); + try testing.expectEqual(4, diffLevenshtein(diffs)); + } + { + var diffs = DiffList{}; + defer diffs.deinit(allocator); + try diffs.appendSlice(allocator, &.{ + Diff.init(.delete, "abc"), + Diff.init(.equal, "xyz"), + Diff.init(.insert, "1234"), + }); + try testing.expectEqual(7, diffLevenshtein(diffs)); + } +} + //| MATCH FUNCTIONS /// Locate the best instance of 'pattern' in 'text' near 'loc'. From 1309b9de8a0d79a9e017c5ca88f45a03bed0c4f3 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 12:09:12 -0400 Subject: [PATCH 079/176] Add comments proving bitap can be changed I tried the adjustment out on the Python3 branch of OG bitap. It works. --- DiffMatchPatch.zig | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index e9f5399..8858aa5 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2082,7 +2082,7 @@ pub fn matchMain( // Shortcut (potentially not guaranteed by the algorithm) // TODO would be good to know what the above means... return 0; - } else if (text.len == 0) { + } else if (text.len == 0 or pattern.len == 0) { // Nothing to match. return null; } else if (loc + pattern.len <= text.len and std.mem.eql(u8, text[loc..pattern.length], pattern)) { @@ -2228,6 +2228,7 @@ fn matchBitap( // TODO decide what to do here: // assert (Match_MaxBits == 0 || pattern.Length <= Match_MaxBits) // : "Pattern too long for this application."; + assert(text.len != 0 and pattern.len != 0); // Initialise the alphabet. var map = try matchAlphabet(allocator, pattern); @@ -2235,17 +2236,18 @@ fn matchBitap( // Highest score beyond which we give up. var threshold = dmp.threshold; // Is there a nearby exact match? (speedup) - var best_loc = std.mem.indexOfPos(u8, text, pattern); - if (best_loc) |best| { - threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), threshold); - } // TODO obviously if we want a speedup here, we do this: - // if (threshold == 0.0) return best_loc; + // if (threshold == 0.0) return best_loc; #proof in comments // We don't have to unwrap best_loc because the retval is ?usize already + // #proof axiom: threshold is between 0.0 and 1.0 (doc comment) + var best_loc = std.mem.indexOfPos(u8, text, loc, pattern); + if (best_loc) |best| { // #proof this returns 0.0 for exact match (see comments in function) + threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + } // What about in the other direction? (speedup) const trunc_text = text[0..@min(loc + pattern.len, text.len)]; best_loc = std.mem.lastIndexOf(u8, trunc_text, pattern); - if (best_loc) |best| { + if (best_loc) |best| { // #proof same here obviously threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); } // Initialise the bit arrays. @@ -2265,6 +2267,7 @@ fn matchBitap( bin_min = 0; bin_mid = bin_max; while (bin_min < bin_mid) { + // #proof lemma: if threshold == 0.0, this never happens if (matchBitapScore(d, loc + bin_mid, loc, pattern) <= threshold) { bin_min = bin_mid; } else { @@ -2277,7 +2280,7 @@ fn matchBitap( var start = @max(1, loc - bin_mid + 1); const finish = @min(loc + bin_mid, text.len) + pattern.len; // No errors below this point, so no errdefer either: - var rd = try allocator.alloc(usize, finish + 2); + var rd: []usize = try allocator.alloc(usize, finish + 2); const dshift: u6 = @intCast(d); rd[finish + 1] = (1 << dshift) - 1; var j = finish; @@ -2298,6 +2301,7 @@ fn matchBitap( const score = matchBitapScore(d, j - 1, loc, pattern); // This match will almost certainly be better than any existing // match. But check anyway. + // #proof: the smoking gun. This can only be equal not less. if (score <= threshold) { // Told you so. threshold = score; @@ -2311,7 +2315,8 @@ fn matchBitap( } } } - } + } // #proof Anything else will do this. + // #proof d + 1 starts at 1, so (see function) this will always break. if (matchBitapScore(d + 1, loc, loc, pattern) > threshold) { // No hope for a (better) match at greater error levels. break; From 20b6ba9706b8f571e4263225633cf11277bd38ba Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 13:47:30 -0400 Subject: [PATCH 080/176] Merge memory-management branch onto more-port --- .gitignore | 1 + DiffMatchPatch.zig | 164 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 153 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 8c9d17e..8571fdf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .zig-cache zig-cache zig-out +kcov-output \ No newline at end of file diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 8858aa5..ccaca80 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -630,7 +630,6 @@ fn diffHalfMatchInternal( const prefix_after = try allocator.dupe(u8, best_short_text_a); errdefer allocator.free(prefix_after); const suffix_after = try allocator.dupe(u8, best_short_text_b); - errdefer allocator.free(suffix_after); const best_common_text = try best_common.toOwnedSlice(allocator); errdefer allocator.free(best_common_text); return .{ @@ -1629,6 +1628,13 @@ fn diffCleanupSemanticScore(one: []const u8, two: []const u8) usize { return 0; } +inline fn boolInt(b: bool) usize { + if (b) + return 1 + else + return 0; +} + /// Reduce the number of edits by eliminating operationally trivial /// equalities. TODO this needs tests pub fn diffCleanupEfficiency( @@ -1638,11 +1644,11 @@ pub fn diffCleanupEfficiency( ) DiffError!void { var changes = false; // Stack of indices where equalities are found. - var equalities = DiffList{}; - defer deinitDiffList(allocator, equalities); + var equalities = std.ArrayList(usize).init(allocator); + defer equalities.deinit(); // Always equal to equalities[equalitiesLength-1][1] - var last_equality = ""; - var pointer: isize = 0; // Index of current position. + var last_equality: []const u8 = ""; + var ipointer: isize = 0; // Index of current position. // Is there an insertion operation before the last equality. var pre_ins = false; // Is there a deletion operation before the last equality. @@ -1651,11 +1657,12 @@ pub fn diffCleanupEfficiency( var post_ins = false; // Is there a deletion operation after the last equality. var post_del = false; - while (pointer < diffs.len) { + while (ipointer < diffs.items.len) { + const pointer: usize = @intCast(ipointer); if (diffs.items[pointer].operation == .equal) { // Equality found. if (diffs.items[pointer].text.len < dmp.diff_edit_cost and (post_ins or post_del)) { // Candidate found. - equalities.Push(pointer); + try equalities.append(pointer); pre_ins = post_ins; pre_del = post_del; last_equality = diffs.items[pointer].text; @@ -1678,10 +1685,10 @@ pub fn diffCleanupEfficiency( // ABXC // AXCD // ABXC - if ((last_equality.Length != 0) and + if ((last_equality.len != 0) and ((pre_ins and pre_del and post_ins and post_del) or - ((last_equality.Length < dmp.diff_edit_cost / 2) and - ((if (pre_ins) 1 else 0) + (if (pre_del) 1 else 0) + (if (post_ins) 1 else 0) + (if (post_del) 1 else 0)) == 3))) + ((last_equality.len < dmp.diff_edit_cost / 2) and + (boolInt(pre_ins) + boolInt(pre_del) + boolInt(post_ins) + boolInt(post_del) == 3)))) { // Duplicate record. try diffs.ensureUnusedCapacity(allocator, 1); @@ -1706,14 +1713,14 @@ pub fn diffCleanupEfficiency( _ = equalities.pop(); } - pointer = if (equalities.items.len > 0) equalities.items[equalities.items.len - 1] else -1; + ipointer = if (equalities.items.len > 0) @intCast(equalities.items[equalities.items.len - 1]) else -1; post_ins = false; post_del = false; } changes = true; } } - pointer += 1; + ipointer += 1; } if (changes) { @@ -4532,3 +4539,136 @@ test diffCleanupSemantic { }, }}); } + +fn testDiffCleanupEfficiency( + allocator: Allocator, + dmp: DiffMatchPatch, + params: struct { + input: []const Diff, + expected: []const Diff, + }, +) !void { + var diffs = try DiffList.initCapacity(allocator, params.input.len); + defer deinitDiffList(allocator, &diffs); + for (params.input) |item| { + diffs.appendAssumeCapacity(.{ .operation = item.operation, .text = try allocator.dupe(u8, item.text) }); + } + try dmp.diffCleanupEfficiency(allocator, &diffs); + + try testing.expectEqualDeep(params.expected, diffs.items); +} + +test "diffCleanupEfficiency" { + const allocator = testing.allocator; + var dmp = DiffMatchPatch{}; + dmp.diff_edit_cost = 4; + { // Null case. + var diffs = DiffList{}; + try dmp.diffCleanupEfficiency(allocator, &diffs); + try testing.expectEqualDeep(DiffList{}, diffs); + } + { // No elimination. + const dslice: []const Diff = &.{ + .{ .operation = .delete, .text = "ab" }, + .{ .operation = .insert, .text = "12" }, + .{ .operation = .equal, .text = "wxyz" }, + .{ .operation = .delete, .text = "cd" }, + .{ .operation = .insert, .text = "34" }, + }; + try testing.checkAllAllocationFailures( + testing.allocator, + testDiffCleanupEfficiency, + .{ + dmp, + .{ .input = dslice, .expected = dslice }, + }, + ); + } + { // Four-edit elimination. + const dslice: []const Diff = &.{ + .{ .operation = .delete, .text = "ab" }, + .{ .operation = .insert, .text = "12" }, + .{ .operation = .equal, .text = "xyz" }, + .{ .operation = .delete, .text = "cd" }, + .{ .operation = .insert, .text = "34" }, + }; + const d_after: []const Diff = &.{ + .{ .operation = .delete, .text = "abxyzcd" }, + .{ .operation = .insert, .text = "12xyz34" }, + }; + try testing.checkAllAllocationFailures( + testing.allocator, + testDiffCleanupEfficiency, + .{ + dmp, + .{ .input = dslice, .expected = d_after }, + }, + ); + } + { // Three-edit elimination. + const dslice: []const Diff = &.{ + .{ .operation = .insert, .text = "12" }, + .{ .operation = .equal, .text = "x" }, + .{ .operation = .delete, .text = "cd" }, + .{ .operation = .insert, .text = "34" }, + }; + const d_after: []const Diff = &.{ + .{ .operation = .delete, .text = "xcd" }, + .{ .operation = .insert, .text = "12x34" }, + }; + try testing.checkAllAllocationFailures( + testing.allocator, + testDiffCleanupEfficiency, + .{ + dmp, + .{ .input = dslice, .expected = d_after }, + }, + ); + } + { // Backpass elimination. + const dslice: []const Diff = &.{ + .{ .operation = .delete, .text = "ab" }, + .{ .operation = .insert, .text = "12" }, + .{ .operation = .equal, .text = "xy" }, + .{ .operation = .insert, .text = "34" }, + .{ .operation = .equal, .text = "z" }, + .{ .operation = .delete, .text = "cd" }, + .{ .operation = .insert, .text = "56" }, + }; + const d_after: []const Diff = &.{ + .{ .operation = .delete, .text = "abxyzcd" }, + .{ .operation = .insert, .text = "12xy34z56" }, + }; + try testing.checkAllAllocationFailures( + testing.allocator, + testDiffCleanupEfficiency, + .{ + dmp, + .{ .input = dslice, .expected = d_after }, + }, + ); + } + { // High cost elimination. + dmp.diff_edit_cost = 5; + const dslice: []const Diff = &.{ + .{ .operation = .delete, .text = "ab" }, + .{ .operation = .insert, .text = "12" }, + .{ .operation = .equal, .text = "wxyz" }, + .{ .operation = .delete, .text = "cd" }, + .{ .operation = .insert, .text = "34" }, + }; + const d_after: []const Diff = &.{ + .{ .operation = .delete, .text = "abwxyzcd" }, + .{ .operation = .insert, .text = "12wxyz34" }, + }; + try testing.checkAllAllocationFailures( + testing.allocator, + testDiffCleanupEfficiency, + .{ + dmp, + .{ .input = dslice, .expected = d_after }, + }, + ); + dmp.diff_edit_cost = 4; + } +} From ef0910b079613c30bf14a31f20c080fcea97a9c6 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 15:10:53 -0400 Subject: [PATCH 081/176] Add test for prettyFormat --- DiffMatchPatch.zig | 95 +++++++++++++++++++++++++++++----------------- 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index ccaca80..b624800 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1843,39 +1843,13 @@ pub fn diffIndex(diffs: DiffList, u_loc: usize) usize { return @intCast(last_chars2 + (loc - last_chars1)); } -test diffIndex { - const dmp = DiffMatchPatch{}; - { - var diffs = try dmp.diff( - testing.allocator, - "The midnight train", - "The blue midnight train", - false, - ); - defer deinitDiffList(testing.allocator, &diffs); - try testing.expectEqual(0, diffIndex(diffs, 0)); - try testing.expectEqual(9, diffIndex(diffs, 4)); - } - { - var diffs = try dmp.diff( - testing.allocator, - "Better still to live and learn", - "Better yet to learn and live", - false, - ); - defer deinitDiffList(testing.allocator, &diffs); - try testing.expectEqual(11, diffIndex(diffs, 13)); - try testing.expectEqual(20, diffIndex(diffs, 21)); - } -} - /// A struct holding bookends for `diffPrittyFormat(diffs)`. /// -/// May include a function taking an allocator and the diff, -/// which shall return the text of the diff, appropriately munged. -/// Note that if the function is provided, all text returned will -/// be freed, so it should always return a copy whether or not -/// edits are needed. +/// May include a function taking an allocator and the Diff, +/// which shall return the text of the Diff, appropriately munged. +/// This allows for tasks like proper HTML escaping. Note that if +/// the function is provided, all text returned will be freed, so +/// it should always return a copy whether or not edits are needed. pub const DiffDecorations = struct { delete_start: []const u8 = "", delete_end: []const u8 = "", @@ -1906,7 +1880,7 @@ pub fn diffPrettyFormat( defer out.deinit(); const writer = out.writer(); _ = try writeDiffPrettyFormat(allocator, writer, diffs, deco); - return out.toOwnedSlice(allocator); + return out.toOwnedSlice(); } /// Pretty-print a diff for output to a terminal. @@ -1924,7 +1898,7 @@ pub fn writeDiffPrettyFormat( deco: DiffDecorations, ) !usize { var written: usize = 0; - for (diffs) |d| { + for (diffs.items) |d| { const text = if (deco.pre_process) |lambda| try lambda(allocator, d) else @@ -1945,7 +1919,7 @@ pub fn writeDiffPrettyFormat( written += try writer.write(text); written += try writer.write(deco.insert_end); }, - .equals => { + .equal => { written += try writer.write(deco.equals_start); written += try writer.write(text); written += try writer.write(deco.equals_end); @@ -4672,3 +4646,56 @@ test "diffCleanupEfficiency" { dmp.diff_edit_cost = 4; } } + +test diffIndex { + const dmp = DiffMatchPatch{}; + { + var diffs = try dmp.diff( + testing.allocator, + "The midnight train", + "The blue midnight train", + false, + ); + defer deinitDiffList(testing.allocator, &diffs); + try testing.expectEqual(0, diffIndex(diffs, 0)); + try testing.expectEqual(9, diffIndex(diffs, 4)); + } + { + var diffs = try dmp.diff( + testing.allocator, + "Better still to live and learn", + "Better yet to learn and live", + false, + ); + defer deinitDiffList(testing.allocator, &diffs); + try testing.expectEqual(11, diffIndex(diffs, 13)); + try testing.expectEqual(20, diffIndex(diffs, 21)); + } +} + +test "diffPrettyFormat" { + const test_deco = DiffDecorations{ + .delete_start = "<+>", + .delete_end = "", + .insert_start = "<->", + .insert_end = "", + .equals_start = "<=>", + .equals_end = "", + }; + const dmp = DiffMatchPatch{}; + const allocator = std.testing.allocator; + var diffs = try dmp.diff( + allocator, + "A thing of beauty is a joy forever", + "Singular beauty is enjoyed forever", + false, + ); + defer deinitDiffList(allocator, &diffs); + try diffCleanupSemantic(allocator, &diffs); + const out_text = try diffPrettyFormat(allocator, diffs, test_deco); + defer allocator.free(out_text); + try testing.expectEqualStrings( + "<+>A thing of<->Singular<=> beauty is <+>a <->en<=>joy<->ed<=> forever", + out_text, + ); +} From 57557c317b6de6b9a9caff18619174dcc8a55034 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 15:14:11 -0400 Subject: [PATCH 082/176] Move before and after test --- DiffMatchPatch.zig | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index b624800..390d92c 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1961,21 +1961,6 @@ pub fn diffAfterText(allocator: Allocator, diffs: DiffList) ![]const u8 { return chars.toOwnedSlice(allocator); } -test "diff before and after text" { - const dmp = DiffMatchPatch{}; - const allocator = testing.allocator; - const before = "The cat in the hat."; - const after = "The bat in the belfry."; - var diffs = try dmp.diff(allocator, before, after, false); - defer deinitDiffList(allocator, &diffs); - const before1 = try diffBeforeText(allocator, diffs); - defer allocator.free(before1); - const after1 = try diffAfterText(allocator, diffs); - defer allocator.free(after1); - try testing.expectEqualStrings(before, before1); - try testing.expectEqualStrings(after, after1); -} - /// /// Compute the Levenshtein distance; the number of inserted, /// deleted or substituted characters. @@ -4647,6 +4632,21 @@ test "diffCleanupEfficiency" { } } +test "diff before and after text" { + const dmp = DiffMatchPatch{}; + const allocator = testing.allocator; + const before = "The cat in the hat."; + const after = "The bat in the belfry."; + var diffs = try dmp.diff(allocator, before, after, false); + defer deinitDiffList(allocator, &diffs); + const before1 = try diffBeforeText(allocator, diffs); + defer allocator.free(before1); + const after1 = try diffAfterText(allocator, diffs); + defer allocator.free(after1); + try testing.expectEqualStrings(before, before1); + try testing.expectEqualStrings(after, after1); +} + test diffIndex { const dmp = DiffMatchPatch{}; { From cc296a8f8f0273cdea9df89f6b708c1e76b89ff7 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 16:01:04 -0400 Subject: [PATCH 083/176] Add test for matchAlphabet With the usual amount of "now that this is compiling" errors. --- DiffMatchPatch.zig | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 390d92c..afb001f 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2324,8 +2324,8 @@ fn matchBitapScore(e: usize, x: usize, loc: usize, pattern: []const u8) f64 { /// Initialise the alphabet for the Bitap algorithm. /// @param pattern The text to encode. /// @return Hash of character locations. -fn matchAlphabet(allocator: Allocator, pattern: []const u8) !std.HashMap(u8, usize) { - var map = std.HashMap(u8, usize).init(allocator); +fn matchAlphabet(allocator: Allocator, pattern: []const u8) !std.AutoHashMap(u8, usize) { + var map = std.AutoHashMap(u8, usize).init(allocator); errdefer map.deinit(); for (pattern) |c| { if (!map.contains(c)) { @@ -2334,7 +2334,7 @@ fn matchAlphabet(allocator: Allocator, pattern: []const u8) !std.HashMap(u8, usi } for (pattern, 0..) |c, i| { const shift: u6 = @intCast(pattern.len - i - 1); - const value: usize = map.get(c) | (1 << shift); + const value: usize = map.get(c).? | (@as(usize, 1) << shift); try map.put(c, value); } return map; @@ -4699,3 +4699,29 @@ test "diffPrettyFormat" { out_text, ); } + +fn testMapSubsetEquality(left: anytype, right: anytype) !void { + var map_iter = left.iterator(); + while (map_iter.next()) |entry| { + const key = entry.key_ptr.*; + const value = entry.value_ptr.*; + try testing.expectEqual(value, right.get(key)); + } +} +test "matchAlphabet" { + var map = std.AutoHashMap(u8, usize).init(testing.allocator); + defer map.deinit(); + try map.put('a', 4); + try map.put('b', 2); + try map.put('c', 1); + var bitap_map = try matchAlphabet(testing.allocator, "abc"); + defer bitap_map.deinit(); + try testMapSubsetEquality(map, bitap_map); + map.clearRetainingCapacity(); + try map.put('a', 37); + try map.put('b', 18); + try map.put('c', 8); + var bitap_map2 = try matchAlphabet(testing.allocator, "abcaba"); + defer bitap_map2.deinit(); + try testMapSubsetEquality(map, bitap_map2); +} From 155b72c7b443abd8094ae735ff81c6e677563794 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 16:22:38 -0400 Subject: [PATCH 084/176] Preliminary testing for matchBitap It has some subtle memory stuff that will be tricky to get right. --- DiffMatchPatch.zig | 84 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 66 insertions(+), 18 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index afb001f..c039f94 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -27,7 +27,7 @@ diff_edit_cost: u16 = 4, /// At what point is no match declared (0.0 = perfection, 1.0 = very loose). /// This defaults to 0.05, on the premise that the library will mostly be /// used in cases where failure is better than a bad patch application. -match_threshold: f32 = 0.05, +match_threshold: f64 = 0.05, /// How far to search for a match (0 = exact location, 1000+ = broad match). /// A match this many characters away from the expected location will add @@ -2177,6 +2177,8 @@ fn ShiftSizeForType(T: type) type { }; } +const sh_one: u64 = 1; + /// Locate the best instance of `pattern` in `text` near `loc` using the /// Bitap algorithm. Returns -1 if no match found. /// @@ -2200,7 +2202,7 @@ fn matchBitap( var map = try matchAlphabet(allocator, pattern); defer map.deinit(); // Highest score beyond which we give up. - var threshold = dmp.threshold; + var threshold = dmp.match_threshold; // Is there a nearby exact match? (speedup) // TODO obviously if we want a speedup here, we do this: // if (threshold == 0.0) return best_loc; #proof in comments @@ -2208,17 +2210,17 @@ fn matchBitap( // #proof axiom: threshold is between 0.0 and 1.0 (doc comment) var best_loc = std.mem.indexOfPos(u8, text, loc, pattern); if (best_loc) |best| { // #proof this returns 0.0 for exact match (see comments in function) - threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), threshold); } // What about in the other direction? (speedup) const trunc_text = text[0..@min(loc + pattern.len, text.len)]; best_loc = std.mem.lastIndexOf(u8, trunc_text, pattern); if (best_loc) |best| { // #proof same here obviously - threshold = @min(matchBitapScore(0, best, loc, pattern), threshold); + threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), threshold); } // Initialise the bit arrays. const shift: u6 = @intCast(pattern.len - 1); - const matchmask = 1 << shift; + const matchmask = sh_one << shift; best_loc = null; var bin_min: usize = undefined; var bin_mid: usize = undefined; @@ -2234,7 +2236,7 @@ fn matchBitap( bin_mid = bin_max; while (bin_min < bin_mid) { // #proof lemma: if threshold == 0.0, this never happens - if (matchBitapScore(d, loc + bin_mid, loc, pattern) <= threshold) { + if (dmp.matchBitapScore(d, loc + bin_mid, loc, pattern) <= threshold) { bin_min = bin_mid; } else { bin_max = bin_mid; @@ -2248,14 +2250,14 @@ fn matchBitap( // No errors below this point, so no errdefer either: var rd: []usize = try allocator.alloc(usize, finish + 2); const dshift: u6 = @intCast(d); - rd[finish + 1] = (1 << dshift) - 1; + rd[finish + 1] = (sh_one << dshift) - 1; var j = finish; while (j >= start) : (j -= 1) { const char_match: usize = if (text.len <= j - 1 or !map.contains(text[j - 1])) // Out of range. 0 else - map.get(text[j - 1]); + map.get(text[j - 1]).?; if (d == 0) { // First pass: exact match. rd[j] = ((rd[j + 1] << 1) | 1) & char_match; @@ -2264,7 +2266,7 @@ fn matchBitap( rd[j] = ((rd[j + 1] << 1) | 1) & char_match | (((last_rd[j + 1] | last_rd[j]) << 1) | 1) | last_rd[j + 1]; } if ((rd[j] & matchmask) != 0) { - const score = matchBitapScore(d, j - 1, loc, pattern); + const score = dmp.matchBitapScore(d, j - 1, loc, pattern); // This match will almost certainly be better than any existing // match. But check anyway. // #proof: the smoking gun. This can only be equal not less. @@ -2272,9 +2274,9 @@ fn matchBitap( // Told you so. threshold = score; best_loc = j - 1; - if (best_loc > loc) { + if (best_loc.? > loc) { // When passing loc, don't exceed our current distance from loc. - start = @max(1, 2 * loc - best_loc); + start = @max(1, 2 * loc - best_loc.?); } else { // Already passed loc, downhill from here on in. break; @@ -2283,7 +2285,7 @@ fn matchBitap( } } // #proof Anything else will do this. // #proof d + 1 starts at 1, so (see function) this will always break. - if (matchBitapScore(d + 1, loc, loc, pattern) > threshold) { + if (dmp.matchBitapScore(d + 1, loc, loc, pattern) > threshold) { // No hope for a (better) match at greater error levels. break; } @@ -2294,31 +2296,77 @@ fn matchBitap( return best_loc; } +fn testMatchBitap( + allocator: Allocator, + dmp: DiffMatchPatch, + params: struct { + text: []const u8, + pattern: []const u8, + loc: usize, + expect: ?usize, + }, +) !void { + const best_loc = try dmp.matchBitap( + allocator, + params.text, + params.pattern, + params.loc, + ); + try testing.expectEqual(params.expect, best_loc); +} + +test "matchBitap" { + var dmp = DiffMatchPatch{}; + dmp.match_distance = 500; + dmp.match_threshold = 0.5; + //assertEquals("match_bitap: Exact match #1.", 5, this.match_bitap("abcdefghijk", "fgh", 5)); + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdefghijk", + .pattern = "fgh", + .loc = 5, + .expect = 5, + }, + }, + ); +} + /// Compute and return the score for a match with e errors and x location. /// @param e Number of errors in match. /// @param x Location of match. /// @param loc Expected location of match. /// @param pattern Pattern being sought. /// @return Overall score for match (0.0 = good, 1.0 = bad). -fn matchBitapScore(e: usize, x: usize, loc: usize, pattern: []const u8) f64 { +fn matchBitapScore( + dmp: DiffMatchPatch, + e: usize, + x: usize, + loc: usize, + pattern: []const u8, +) f64 { // shortcut? TODO, proof in comments // if (e == 0 and x == loc) return 0.0; - const e_float: f32 = @floatFromInt(e); - const len_float: f32 = @floatFromInt(pattern.len); + const e_float: f64 = @floatFromInt(e); + const len_float: f64 = @floatFromInt(pattern.len); // if e == 0, accuracy == 0: 0/x = 0 const accuracy = e_float / len_float; // if loc == x, proximity == 0 const proximity = if (loc >= x) loc - x else x - loc; - if (@This().match_distance == 0) { + if (dmp.match_distance == 0) { // Dodge divide by zero if (proximity == 0) // therefore this returns 0 return accuracy else return 1.0; } - const float_match: f64 = @floatFromInt(@This().match_distance); + const float_match: f64 = @floatFromInt(dmp.match_distance); + const float_proximity: f64 = @floatFromInt(proximity); // or this is 0 + 0/f_m aka 0 - return accuracy + (proximity / float_match); + return accuracy + (float_proximity / float_match); } /// Initialise the alphabet for the Bitap algorithm. From 13cd97373cfe43c3e6f29f2b709fb95c5a56d7d0 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 16:38:55 -0400 Subject: [PATCH 085/176] Not actually that subtle Just missed an exit from the loop which wasn't freeing memory. So it goes. --- DiffMatchPatch.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index c039f94..5a650b0 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2249,6 +2249,7 @@ fn matchBitap( const finish = @min(loc + bin_mid, text.len) + pattern.len; // No errors below this point, so no errdefer either: var rd: []usize = try allocator.alloc(usize, finish + 2); + errdefer allocator.free(rd); const dshift: u6 = @intCast(d); rd[finish + 1] = (sh_one << dshift) - 1; var j = finish; @@ -2287,6 +2288,7 @@ fn matchBitap( // #proof d + 1 starts at 1, so (see function) this will always break. if (dmp.matchBitapScore(d + 1, loc, loc, pattern) > threshold) { // No hope for a (better) match at greater error levels. + allocator.free(rd); break; } allocator.free(last_rd); From 236179a8431a6839a35091464468879f5d44bbf1 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 17:47:30 -0400 Subject: [PATCH 086/176] Bitap incorrect somewhere Always a fun story. --- DiffMatchPatch.zig | 97 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 81 insertions(+), 16 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 5a650b0..b0708ed 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2202,7 +2202,7 @@ fn matchBitap( var map = try matchAlphabet(allocator, pattern); defer map.deinit(); // Highest score beyond which we give up. - var threshold = dmp.match_threshold; + var score_threshold = dmp.match_threshold; // Is there a nearby exact match? (speedup) // TODO obviously if we want a speedup here, we do this: // if (threshold == 0.0) return best_loc; #proof in comments @@ -2210,22 +2210,27 @@ fn matchBitap( // #proof axiom: threshold is between 0.0 and 1.0 (doc comment) var best_loc = std.mem.indexOfPos(u8, text, loc, pattern); if (best_loc) |best| { // #proof this returns 0.0 for exact match (see comments in function) - threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), threshold); + score_threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), score_threshold); } // What about in the other direction? (speedup) const trunc_text = text[0..@min(loc + pattern.len, text.len)]; best_loc = std.mem.lastIndexOf(u8, trunc_text, pattern); if (best_loc) |best| { // #proof same here obviously - threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), threshold); + score_threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), score_threshold); } // Initialise the bit arrays. const shift: u6 = @intCast(pattern.len - 1); const matchmask = sh_one << shift; best_loc = null; - var bin_min: usize = undefined; - var bin_mid: usize = undefined; - var bin_max = pattern.len + text.len; - // null last_rd to simplying freeing memory + // Zig is very insistent about integer width and signedness. + const i_textlen: isize = @intCast(text.len); + const i_patlen: isize = @intCast(pattern.len); + + const i_loc: isize = @intCast(loc); + var bin_min: isize = undefined; + var bin_mid: isize = undefined; + var bin_max: isize = i_patlen + i_textlen; + // null last_rd to simplify freeing memory var last_rd: []usize = try allocator.alloc(usize, 0); errdefer allocator.free(last_rd); for (0..pattern.len) |d| { @@ -2236,19 +2241,20 @@ fn matchBitap( bin_mid = bin_max; while (bin_min < bin_mid) { // #proof lemma: if threshold == 0.0, this never happens - if (dmp.matchBitapScore(d, loc + bin_mid, loc, pattern) <= threshold) { + if (dmp.matchBitapScore(d, @intCast(i_loc + bin_mid), loc, pattern) <= score_threshold) { bin_min = bin_mid; } else { bin_max = bin_mid; } - bin_mid = (bin_max - bin_min) / 2 + bin_min; + bin_mid = @divTrunc(bin_max - bin_min, 2 + bin_min); } // Use the result from this iteration as the maximum for the next. bin_max = bin_mid; - var start = @max(1, loc - bin_mid + 1); - const finish = @min(loc + bin_mid, text.len) + pattern.len; + var start: usize = @intCast(@max(1, i_loc - bin_mid + 1)); + const finish: usize = @intCast(@min(i_loc + bin_mid, i_textlen) + i_patlen); // No errors below this point, so no errdefer either: var rd: []usize = try allocator.alloc(usize, finish + 2); + @memset(rd, 0); // XXX might not help, decide errdefer allocator.free(rd); const dshift: u6 = @intCast(d); rd[finish + 1] = (sh_one << dshift) - 1; @@ -2271,9 +2277,9 @@ fn matchBitap( // This match will almost certainly be better than any existing // match. But check anyway. // #proof: the smoking gun. This can only be equal not less. - if (score <= threshold) { + if (score <= score_threshold) { // Told you so. - threshold = score; + score_threshold = score; best_loc = j - 1; if (best_loc.? > loc) { // When passing loc, don't exceed our current distance from loc. @@ -2286,7 +2292,7 @@ fn matchBitap( } } // #proof Anything else will do this. // #proof d + 1 starts at 1, so (see function) this will always break. - if (dmp.matchBitapScore(d + 1, loc, loc, pattern) > threshold) { + if (dmp.matchBitapScore(d + 1, loc, loc, pattern) > score_threshold) { // No hope for a (better) match at greater error levels. allocator.free(rd); break; @@ -2321,7 +2327,23 @@ test "matchBitap" { var dmp = DiffMatchPatch{}; dmp.match_distance = 500; dmp.match_threshold = 0.5; - //assertEquals("match_bitap: Exact match #1.", 5, this.match_bitap("abcdefghijk", "fgh", 5)); + // match_bitap: Exact match #1. + if (false) { + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdefghijk", + .pattern = "fgh", + .loc = 5, + .expect = 5, + }, + }, + ); + } + // match_bitap: Exact match #2. try testing.checkAllAllocationFailures( testing.allocator, testMatchBitap, @@ -2330,11 +2352,50 @@ test "matchBitap" { .{ .text = "abcdefghijk", .pattern = "fgh", - .loc = 5, + .loc = 0, .expect = 5, }, }, ); + // assertEquals("match_bitap: Exact match #1.", 5, this.match_bitap("abcdefghijk", "fgh", 5)); + // + // assertEquals("match_bitap: Exact match #2.", 5, this.match_bitap("abcdefghijk", "fgh", 0)); + // + // assertEquals("match_bitap: Fuzzy match #1.", 4, this.match_bitap("abcdefghijk", "efxhi", 0)); + // + // assertEquals("match_bitap: Fuzzy match #2.", 2, this.match_bitap("abcdefghijk", "cdefxyhijk", 5)); + // + // assertEquals("match_bitap: Fuzzy match #3.", -1, this.match_bitap("abcdefghijk", "bxy", 1)); + // + // assertEquals("match_bitap: Overflow.", 2, this.match_bitap("123456789xx0", "3456789x0", 2)); + // + // assertEquals("match_bitap: Before start match.", 0, this.match_bitap("abcdef", "xxabc", 4)); + // + // assertEquals("match_bitap: Beyond end match.", 3, this.match_bitap("abcdef", "defyy", 4)); + // + // assertEquals("match_bitap: Oversized pattern.", 0, this.match_bitap("abcdef", "xabcdefy", 0)); + // + // this.Match_Threshold = 0.4f; + // assertEquals("match_bitap: Threshold #1.", 4, this.match_bitap("abcdefghijk", "efxyhi", 1)); + // + // this.Match_Threshold = 0.3f; + // assertEquals("match_bitap: Threshold #2.", -1, this.match_bitap("abcdefghijk", "efxyhi", 1)); + // + // this.Match_Threshold = 0.0f; + // assertEquals("match_bitap: Threshold #3.", 1, this.match_bitap("abcdefghijk", "bcdef", 1)); + // + // this.Match_Threshold = 0.5f; + // assertEquals("match_bitap: Multiple select #1.", 0, this.match_bitap("abcdexyzabcde", "abccde", 3)); + // + // assertEquals("match_bitap: Multiple select #2.", 8, this.match_bitap("abcdexyzabcde", "abccde", 5)); + // + // this.Match_Distance = 10; // Strict location. + // assertEquals("match_bitap: Distance test #1.", -1, this.match_bitap("abcdefghijklmnopqrstuvwxyz", "abcdefg", 24)); + // + // assertEquals("match_bitap: Distance test #2.", 0, this.match_bitap("abcdefghijklmnopqrstuvwxyz", "abcdxxefg", 1)); + // + // this.Match_Distance = 1000; // Loose location. + // assertEquals("match_bitap: Distance test #3.", 0, this.match_bitap("abcdefghijklmnopqrstuvwxyz", "abcdefg", 24)); } /// Compute and return the score for a match with e errors and x location. @@ -2358,6 +2419,10 @@ fn matchBitapScore( const accuracy = e_float / len_float; // if loc == x, proximity == 0 const proximity = if (loc >= x) loc - x else x - loc; + // TODO this seems obviously equivalent but wtf, debugging + // const ix: isize = @intCast(x); + // const proximity = @abs(i_loc - ix); + // const i_loc: isize = @intCast(loc); if (dmp.match_distance == 0) { // Dodge divide by zero if (proximity == 0) // therefore this returns 0 From 08fe19b68e885a26d02cfe38f7d60a9474587fd1 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 18:19:57 -0400 Subject: [PATCH 087/176] Screwed up operation order, thanks @divTrunc This is like, my one heretical Zig opinion: it could pick a lane and truncate toward negative infinity by default for signed ints. Whatever, bug squashed, carrying on. --- DiffMatchPatch.zig | 51 ++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index b0708ed..f0986c6 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2246,7 +2246,7 @@ fn matchBitap( } else { bin_max = bin_mid; } - bin_mid = @divTrunc(bin_max - bin_min, 2 + bin_min); + bin_mid = @divTrunc(bin_max - bin_min, 2) + bin_min; } // Use the result from this iteration as the maximum for the next. bin_max = bin_mid; @@ -2283,7 +2283,8 @@ fn matchBitap( best_loc = j - 1; if (best_loc.? > loc) { // When passing loc, don't exceed our current distance from loc. - start = @max(1, 2 * loc - best_loc.?); + const i_best_loc: isize = @intCast(best_loc.?); + start = @max(1, 2 * i_loc - i_best_loc); } else { // Already passed loc, downhill from here on in. break; @@ -2328,21 +2329,20 @@ test "matchBitap" { dmp.match_distance = 500; dmp.match_threshold = 0.5; // match_bitap: Exact match #1. - if (false) { - try testing.checkAllAllocationFailures( - testing.allocator, - testMatchBitap, + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, .{ - dmp, - .{ - .text = "abcdefghijk", - .pattern = "fgh", - .loc = 5, - .expect = 5, - }, + .text = "abcdefghijk", + .pattern = "fgh", + .loc = 5, + .expect = 5, }, - ); - } + }, + ); + // match_bitap: Exact match #2. try testing.checkAllAllocationFailures( testing.allocator, @@ -2357,11 +2357,22 @@ test "matchBitap" { }, }, ); - // assertEquals("match_bitap: Exact match #1.", 5, this.match_bitap("abcdefghijk", "fgh", 5)); - // - // assertEquals("match_bitap: Exact match #2.", 5, this.match_bitap("abcdefghijk", "fgh", 0)); - // - // assertEquals("match_bitap: Fuzzy match #1.", 4, this.match_bitap("abcdefghijk", "efxhi", 0)); + // Fuzzy match #1 + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdefghijk", + .pattern = "efxhi", + .loc = 0, + .expect = 4, + }, + }, + ); + + // assertEquals("match_bitap: .", 4, this.match_bitap("abcdefghijk", "efxhi", 0)); // // assertEquals("match_bitap: Fuzzy match #2.", 2, this.match_bitap("abcdefghijk", "cdefxyhijk", 5)); // From 4816085cdd869c91d79835fc0ddd9dd6bffab4ed Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 19:00:27 -0400 Subject: [PATCH 088/176] Add remaining matchBitap tests Even though all the Python tests passed when I made the change to more efficient matchBitap calculations, I'm going to put off doing them here until I have the rest of the code tested. --- DiffMatchPatch.zig | 360 ++++++++++++++++++++++++++++++++------------- 1 file changed, 256 insertions(+), 104 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index f0986c6..53d26c0 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2305,110 +2305,6 @@ fn matchBitap( return best_loc; } -fn testMatchBitap( - allocator: Allocator, - dmp: DiffMatchPatch, - params: struct { - text: []const u8, - pattern: []const u8, - loc: usize, - expect: ?usize, - }, -) !void { - const best_loc = try dmp.matchBitap( - allocator, - params.text, - params.pattern, - params.loc, - ); - try testing.expectEqual(params.expect, best_loc); -} - -test "matchBitap" { - var dmp = DiffMatchPatch{}; - dmp.match_distance = 500; - dmp.match_threshold = 0.5; - // match_bitap: Exact match #1. - try testing.checkAllAllocationFailures( - testing.allocator, - testMatchBitap, - .{ - dmp, - .{ - .text = "abcdefghijk", - .pattern = "fgh", - .loc = 5, - .expect = 5, - }, - }, - ); - - // match_bitap: Exact match #2. - try testing.checkAllAllocationFailures( - testing.allocator, - testMatchBitap, - .{ - dmp, - .{ - .text = "abcdefghijk", - .pattern = "fgh", - .loc = 0, - .expect = 5, - }, - }, - ); - // Fuzzy match #1 - try testing.checkAllAllocationFailures( - testing.allocator, - testMatchBitap, - .{ - dmp, - .{ - .text = "abcdefghijk", - .pattern = "efxhi", - .loc = 0, - .expect = 4, - }, - }, - ); - - // assertEquals("match_bitap: .", 4, this.match_bitap("abcdefghijk", "efxhi", 0)); - // - // assertEquals("match_bitap: Fuzzy match #2.", 2, this.match_bitap("abcdefghijk", "cdefxyhijk", 5)); - // - // assertEquals("match_bitap: Fuzzy match #3.", -1, this.match_bitap("abcdefghijk", "bxy", 1)); - // - // assertEquals("match_bitap: Overflow.", 2, this.match_bitap("123456789xx0", "3456789x0", 2)); - // - // assertEquals("match_bitap: Before start match.", 0, this.match_bitap("abcdef", "xxabc", 4)); - // - // assertEquals("match_bitap: Beyond end match.", 3, this.match_bitap("abcdef", "defyy", 4)); - // - // assertEquals("match_bitap: Oversized pattern.", 0, this.match_bitap("abcdef", "xabcdefy", 0)); - // - // this.Match_Threshold = 0.4f; - // assertEquals("match_bitap: Threshold #1.", 4, this.match_bitap("abcdefghijk", "efxyhi", 1)); - // - // this.Match_Threshold = 0.3f; - // assertEquals("match_bitap: Threshold #2.", -1, this.match_bitap("abcdefghijk", "efxyhi", 1)); - // - // this.Match_Threshold = 0.0f; - // assertEquals("match_bitap: Threshold #3.", 1, this.match_bitap("abcdefghijk", "bcdef", 1)); - // - // this.Match_Threshold = 0.5f; - // assertEquals("match_bitap: Multiple select #1.", 0, this.match_bitap("abcdexyzabcde", "abccde", 3)); - // - // assertEquals("match_bitap: Multiple select #2.", 8, this.match_bitap("abcdexyzabcde", "abccde", 5)); - // - // this.Match_Distance = 10; // Strict location. - // assertEquals("match_bitap: Distance test #1.", -1, this.match_bitap("abcdefghijklmnopqrstuvwxyz", "abcdefg", 24)); - // - // assertEquals("match_bitap: Distance test #2.", 0, this.match_bitap("abcdefghijklmnopqrstuvwxyz", "abcdxxefg", 1)); - // - // this.Match_Distance = 1000; // Loose location. - // assertEquals("match_bitap: Distance test #3.", 0, this.match_bitap("abcdefghijklmnopqrstuvwxyz", "abcdefg", 24)); -} - /// Compute and return the score for a match with e errors and x location. /// @param e Number of errors in match. /// @param x Location of match. @@ -4851,3 +4747,259 @@ test "matchAlphabet" { defer bitap_map2.deinit(); try testMapSubsetEquality(map, bitap_map2); } + +fn testMatchBitap( + allocator: Allocator, + dmp: DiffMatchPatch, + params: struct { + text: []const u8, + pattern: []const u8, + loc: usize, + expect: ?usize, + }, +) !void { + const best_loc = try dmp.matchBitap( + allocator, + params.text, + params.pattern, + params.loc, + ); + try testing.expectEqual(params.expect, best_loc); +} + +test matchBitap { + var dmp = DiffMatchPatch{}; + dmp.match_distance = 500; + dmp.match_threshold = 0.5; + // Exact match #1. + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdefghijk", + .pattern = "fgh", + .loc = 5, + .expect = 5, + }, + }, + ); + // Exact match #2. + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdefghijk", + .pattern = "fgh", + .loc = 0, + .expect = 5, + }, + }, + ); + // Fuzzy match #1 + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdefghijk", + .pattern = "efxhi", + .loc = 0, + .expect = 4, + }, + }, + ); + // Fuzzy match #2. + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdefghijk", + .pattern = "cdefxyhijk", + .loc = 5, + .expect = 2, + }, + }, + ); + // Fuzzy match #3. + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdefghijk", + .pattern = "bxy", + .loc = 1, + .expect = null, + }, + }, + ); + // Overflow. + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "123456789xx0", + .pattern = "3456789x0", + .loc = 2, + .expect = 2, + }, + }, + ); + //Before start match. + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdef", + .pattern = "xxabc", + .loc = 4, + .expect = 0, + }, + }, + ); + // + // Beyond end match. + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdef", + .pattern = "defyy", + .loc = 4, + .expect = 3, + }, + }, + ); + // Oversized pattern. + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdef", + .pattern = "xabcdefy", + .loc = 0, + .expect = 0, + }, + }, + ); + dmp.match_threshold = 0.4; + // Threshold #1. + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdefghijk", + .pattern = "efxyhi", + .loc = 1, + .expect = 4, + }, + }, + ); + dmp.match_threshold = 0.3; + // Threshold #2. + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdefghijk", + .pattern = "efxyhi", + .loc = 1, + .expect = null, + }, + }, + ); + dmp.match_threshold = 0.0; + // Threshold #3. + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdefghijk", + .pattern = "bcdef", + .loc = 1, + .expect = 1, + }, + }, + ); + dmp.match_threshold = 0.5; + // Multiple select #1. + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdexyzabcde", + .pattern = "abccde", + .loc = 5, + .expect = 8, + }, + }, + ); + dmp.match_distance = 10; // Strict location. + // Distance test #1. + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdefghijklmnopqrstuvwxyz", + .pattern = "abcdefg", + .loc = 1, + .expect = 0, + }, + }, + ); + // Distance test #2. + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdefghijklmnopqrstuvwxyz", + .pattern = "abcdxxefg", + .loc = 1, + .expect = 0, + }, + }, + ); + dmp.match_distance = 1000; // Loose location. + // Distance test #3. + try testing.checkAllAllocationFailures( + testing.allocator, + testMatchBitap, + .{ + dmp, + .{ + .text = "abcdefghijklmnopqrstuvwxyz", + .pattern = "abcdefg", + .loc = 24, + .expect = 0, + }, + }, + ); +} From 3931c4ab8922ce2053de122da080b006040e0c4c Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 19:54:03 -0400 Subject: [PATCH 089/176] Add tests for matchMain --- DiffMatchPatch.zig | 72 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 53d26c0..7f76ace 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2041,17 +2041,17 @@ pub fn matchMain( text: []const u8, pattern: []const u8, passed_loc: usize, -) ?usize { +) !?usize { // Clamp the loc to fit within text. const loc = @min(passed_loc, text.len); if (std.mem.eql(u8, text, pattern)) { // Shortcut (potentially not guaranteed by the algorithm) // TODO would be good to know what the above means... return 0; - } else if (text.len == 0 or pattern.len == 0) { + } else if (text.len == 0) { // Nothing to match. return null; - } else if (loc + pattern.len <= text.len and std.mem.eql(u8, text[loc..pattern.length], pattern)) { + } else if (loc + pattern.len <= text.len and std.mem.eql(u8, text[loc .. loc + pattern.len], pattern)) { // Perfect match at the perfect spot! (Includes case of null pattern) return loc; } else { @@ -2254,7 +2254,6 @@ fn matchBitap( const finish: usize = @intCast(@min(i_loc + bin_mid, i_textlen) + i_patlen); // No errors below this point, so no errdefer either: var rd: []usize = try allocator.alloc(usize, finish + 2); - @memset(rd, 0); // XXX might not help, decide errdefer allocator.free(rd); const dshift: u6 = @intCast(d); rd[finish + 1] = (sh_one << dshift) - 1; @@ -2326,10 +2325,6 @@ fn matchBitapScore( const accuracy = e_float / len_float; // if loc == x, proximity == 0 const proximity = if (loc >= x) loc - x else x - loc; - // TODO this seems obviously equivalent but wtf, debugging - // const ix: isize = @intCast(x); - // const proximity = @abs(i_loc - ix); - // const i_loc: isize = @intCast(loc); if (dmp.match_distance == 0) { // Dodge divide by zero if (proximity == 0) // therefore this returns 0 @@ -2390,7 +2385,7 @@ fn matchAlphabetImproved(allocator: Allocator, pattern: []const u8, UIntSize: ty /// /// @param patch The patch to grow. /// @param text Source text. -fn patchAddContext( // XXX pick it up from here +fn patchAddContext( dmp: DiffMatchPatch, allocator: Allocator, patch: *Patch, @@ -5003,3 +4998,62 @@ test matchBitap { }, ); } + +test matchMain { + var dmp = DiffMatchPatch{}; + dmp.match_threshold = 0.5; + dmp.match_distance = 100; + const allocator = testing.allocator; + // Equality. + try testing.expectEqual(0, dmp.matchMain( + allocator, + "abcdefg", + "abcdefg", + 1000, + )); + // Null text + try testing.expectEqual(null, dmp.matchMain( + allocator, + "", + "abcdefg", + 1, + )); + // Null pattern. + try testing.expectEqual(3, dmp.matchMain( + allocator, + "abcdefg", + "", + 3, + )); + // Exact match. + try testing.expectEqual(3, dmp.matchMain( + allocator, + "abcdefg", + "de", + 3, + )); + // Beyond end match. + try testing.expectEqual(3, dmp.matchMain( + allocator, + "abcdef", + "defy", + 4, + )); + + // Oversized pattern. + try testing.expectEqual(0, dmp.matchMain( + allocator, + "abcdef", + "abcdefy", + 0, + )); + dmp.match_threshold = 0.7; + // Complex match. + try testing.expectEqual(4, dmp.matchMain( + allocator, + "I am the very model of a modern major general.", + " that berry ", + 5, + )); + dmp.match_threshold = 0.5; +} From 8f7b83cf51bc36541f580dfe6b36bc3b6a06b311 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 20:47:44 -0400 Subject: [PATCH 090/176] Patch writer test --- DiffMatchPatch.zig | 57 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 7f76ace..c8756ea 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -150,7 +150,7 @@ pub const Patch = struct { } pub fn deinit(patch: *Patch, allocator: Allocator) void { - deinitDiffList(allocator, patch.diffs); + deinitDiffList(allocator, &patch.diffs); } /// Emit patch in Unidiff format, as specifified here: @@ -163,7 +163,7 @@ pub const Patch = struct { var text_array = std.ArrayList(u8).init(allocator); defer text_array.deinit(); const writer = text_array.writer(); - try patch.writeText(writer, patch); + try patch.writeText(writer); return text_array.toOwnedSlice(); } @@ -171,7 +171,7 @@ pub const Patch = struct { /// Stream textual patch representation to Writer. See `asText` /// for more information. - pub fn writeText(writer: anytype, patch: Patch) !void { + pub fn writeText(patch: Patch, writer: anytype) !void { // Write header. _ = try writer.write(PATCH_HEAD); // Stream coordinates @@ -190,15 +190,15 @@ pub const Patch = struct { } else { try format(writer, "{d},{d}", .{ patch.start2 + 1, patch.length2 }); } - _ = writer.write(PATCH_TAIL); + _ = try writer.write(PATCH_TAIL); // Escape the body of the patch with %xx notation. - for (patch.diffs) |a_diff| { + for (patch.diffs.items) |a_diff| { switch (a_diff.operation) { .insert => try writer.writeByte('+'), .delete => try writer.writeByte('-'), - .equal => try writer.writeByte('='), + .equal => try writer.writeByte(' '), } - _ = try writeUriEncoded(writer, diff.text); + _ = try writeUriEncoded(writer, a_diff.text); try writer.writeByte('\n'); } return; @@ -5057,3 +5057,46 @@ test matchMain { )); dmp.match_threshold = 0.5; } + +test "patch to string" { + // + var p: Patch = Patch{ + .start1 = 20, + .start2 = 21, + .length1 = 18, + .length2 = 17, + .diffs = try sliceToDiffList(testing.allocator, &.{ + .{ .operation = .equal, .text = "jump" }, + .{ .operation = .delete, .text = "s" }, + .{ .operation = .insert, .text = "ed" }, + .{ .operation = .equal, .text = " over " }, + .{ .operation = .delete, .text = "the" }, + .{ .operation = .insert, .text = "a" }, + .{ .operation = .equal, .text = "\nlaz" }, + }), + }; + defer p.deinit(testing.allocator); + const strp = "@@ -21,18 +22,17 @@\n jump\n-s\n+ed\n over \n-the\n+a\n %0Alaz\n"; + const patch_str = try p.asText(testing.allocator); + defer testing.allocator.free(patch_str); + try testing.expectEqualStrings(strp, patch_str); +} + +//public void patch_patchObjTest() { +// // Patch Object. +// Patch p = new Patch(); +// p.start1 = 20; +// p.start2 = 21; +// p.length1 = 18; +// p.length2 = 17; +// p.diffs = new List { +// new Diff(Operation.EQUAL, "jump"), +// new Diff(Operation.DELETE, "s"), +// new Diff(Operation.INSERT, "ed"), +// new Diff(Operation.EQUAL, " over "), +// new Diff(Operation.DELETE, "the"), +// new Diff(Operation.INSERT, "a"), +// new Diff(Operation.EQUAL, "\nlaz")}; +// string strp = "@@ -21,18 +22,17 @@\n jump\n-s\n+ed\n over \n-the\n+a\n %0alaz\n"; +// assertEquals("Patch: toString.", strp, p.ToString()); +//} From 9975da5630bdf4d1d8f6872637e716ed177a95ad Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 21:35:00 -0400 Subject: [PATCH 091/176] First round-trip patch test passes --- DiffMatchPatch.zig | 125 +++++++++++++++++++++++++-------------------- 1 file changed, 69 insertions(+), 56 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index c8756ea..babf953 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -72,6 +72,13 @@ fn freeRangeDiffList( } } +pub fn deinitPatchList(allocator: Allocator, patches: *PatchList) void { + defer patches.deinit(allocator); + for (patches.items) |*a_patch| { + deinitDiffList(allocator, &a_patch.diffs); + } +} + /// DMP with default configuration options pub const default = DiffMatchPatch{}; @@ -123,16 +130,6 @@ pub const Patch = struct { start2: usize = 0, length2: usize = 0, - pub fn toString(patch: Patch) ![]const u8 { - // TODO - _ = patch; - } - - pub fn writeTo(writer: anytype) !usize { - // TODO - _ = writer; - } - /// Make a clone of the Patch, including all Diffs. pub fn clone(patch: Patch, allocator: Allocator) !Patch { var new_diffs = DiffList{}; @@ -3018,7 +3015,7 @@ fn patchListClone(allocator: Allocator, patches: PatchList) !PatchList { /// @param patches List of Patch objects. /// @return Text representation of patches. pub fn patchToText(allocator: Allocator, patches: PatchList) ![]const u8 { - const text_array = try std.ArrayList(u8).init(allocator); + var text_array = std.ArrayList(u8).init(allocator); defer text_array.deinit(); const writer = text_array.writer(); try writePatch(writer, patches); @@ -3027,8 +3024,8 @@ pub fn patchToText(allocator: Allocator, patches: PatchList) ![]const u8 { /// Stream a `PatchList` to the provided Writer. pub fn writePatch(writer: anytype, patches: PatchList) !void { - for (patches) |a_patch| { - try a_patch.writePatch(writer); + for (patches.items) |a_patch| { + try a_patch.writeText(writer); } } @@ -3041,34 +3038,36 @@ pub fn patchFromText(allocator: Allocator, text: []const u8) !PatchList { if (text.len == 0) return PatchList{}; var patches = PatchList{}; errdefer patches.deinit(allocator); - var cursor = 0; + var cursor: usize = 0; while (cursor < text.len) { // TODO catch BadPatchString here and print diagnostic + try patches.ensureUnusedCapacity(allocator, 1); const cursor_delta, const patch = try patchFromHeader(allocator, text[cursor..]); cursor += cursor_delta; - try patches.append(allocator, patch); + patches.appendAssumeCapacity(patch); } return patches; } fn countDigits(text: []const u8) usize { - var idx = 0; + var idx: usize = 0; while (std.ascii.isDigit(text[idx])) : (idx += 1) {} return idx; } fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patch } { - var patch = Patch{}; + var patch = Patch{ .diffs = DiffList{} }; errdefer patch.deinit(allocator); var cursor: usize = undefined; if (std.mem.eql(u8, text[0..4], PATCH_HEAD)) { // Parse location and length in before text + const count = 4 + countDigits(text[4..]); patch.start1 = std.fmt.parseInt( usize, - text[4..], + text[4..count], 10, ) catch return error.BadPatchString; - cursor = 4 + countDigits(text[4..]); + cursor = count; assert(cursor > 4); if (text[cursor] != ',') { cursor += 1; @@ -3076,12 +3075,12 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc patch.length1 = 1; } else { cursor += 1; + const delta = countDigits(text[cursor..]); patch.length1 = std.fmt.parseInt( usize, - text[cursor..], + text[cursor .. cursor + delta], 10, ) catch return error.BadPatchString; - const delta = countDigits(text[cursor..]); assert(delta > 0); cursor += delta; if (patch.length1 != 0) { @@ -3092,13 +3091,13 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc // Parse location and length in after text. if (text[cursor] == ' ' and text[cursor + 1] == '+') { cursor += 2; + const delta1 = countDigits(text[cursor..]); + assert(delta1 > 0); patch.start2 = std.fmt.parseInt( usize, - text[cursor..], + text[cursor .. cursor + delta1], 10, ) catch return error.BadPatchString; - const delta1 = 4 + countDigits(text[4..]); - assert(delta1 > 0); cursor += delta1; if (text[cursor] != ',') { cursor += 1; @@ -3106,13 +3105,13 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc patch.length2 = 1; } else { cursor += 1; + const delta2 = countDigits(text[cursor..]); + assert(delta2 > 1); patch.length2 = std.fmt.parseInt( usize, - text[cursor..], + text[cursor .. cursor + delta2], 10, ) catch return error.BadPatchString; - const delta2 = countDigits(text[cursor..]); - assert(delta2 > 1); cursor += delta2; if (patch.length2 != 0) { patch.start2 -= 1; @@ -3123,7 +3122,7 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc cursor += 4; } else return error.BadPatchString; // Eat the diffs - const patch_lines = std.mem.splitScalar( + var patch_lines = std.mem.splitScalar( u8, text[cursor..], '\n', @@ -3135,7 +3134,12 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc if (line.len == 0) continue; // Microsoft encodes spaces as +, we don't, so we don't need this: // line = line.Replace("+", "%2b"); - const diff_line = try decodeUri(allocator, line) catch return error.BadPatchString; + const diff_line = decodeUri(allocator, line[1..]) catch |e| { + switch (e) { + error.OutOfMemory => return e, + else => return error.BadPatchString, + } + }; errdefer allocator.free(diff_line); switch (line[0]) { '+' => { // Insertion @@ -3178,11 +3182,11 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc /// Decode our URI-esque escaping fn decodeUri(allocator: Allocator, line: []const u8) ![]const u8 { - if (std.mem.indexOf(u8, line, '%')) |first| { + if (std.mem.indexOf(u8, line, "%")) |first| { // Text to decode. // Result will always be shorter than line: var new_line = try std.ArrayList(u8).initCapacity(allocator, line.len); - defer new_line.init; + defer new_line.deinit(); try new_line.appendSlice(line[0..first]); var out_buf: [1]u8 = .{0}; var codeunit = std.fmt.hexToBytes( @@ -3191,8 +3195,8 @@ fn decodeUri(allocator: Allocator, line: []const u8) ![]const u8 { ) catch return error.BadPatchString; try new_line.append(codeunit[0]); var cursor = first + 3; - while (std.mem.indexOf(u8, line[cursor..], '%')) |next| { - codeunit = try std.fmt.hexToBytes( + while (std.mem.indexOf(u8, line[cursor..], "%")) |next| { + codeunit = std.fmt.hexToBytes( &out_buf, line[next + 1 .. next + 3], ) catch return error.BadPatchString; @@ -5058,14 +5062,14 @@ test matchMain { dmp.match_threshold = 0.5; } -test "patch to string" { +fn testPatchToText(allocator: Allocator) !void { // var p: Patch = Patch{ .start1 = 20, .start2 = 21, .length1 = 18, .length2 = 17, - .diffs = try sliceToDiffList(testing.allocator, &.{ + .diffs = try sliceToDiffList(allocator, &.{ .{ .operation = .equal, .text = "jump" }, .{ .operation = .delete, .text = "s" }, .{ .operation = .insert, .text = "ed" }, @@ -5075,28 +5079,37 @@ test "patch to string" { .{ .operation = .equal, .text = "\nlaz" }, }), }; - defer p.deinit(testing.allocator); + defer p.deinit(allocator); const strp = "@@ -21,18 +22,17 @@\n jump\n-s\n+ed\n over \n-the\n+a\n %0Alaz\n"; - const patch_str = try p.asText(testing.allocator); - defer testing.allocator.free(patch_str); + const patch_str = try p.asText(allocator); + defer allocator.free(patch_str); try testing.expectEqualStrings(strp, patch_str); } -//public void patch_patchObjTest() { -// // Patch Object. -// Patch p = new Patch(); -// p.start1 = 20; -// p.start2 = 21; -// p.length1 = 18; -// p.length2 = 17; -// p.diffs = new List { -// new Diff(Operation.EQUAL, "jump"), -// new Diff(Operation.DELETE, "s"), -// new Diff(Operation.INSERT, "ed"), -// new Diff(Operation.EQUAL, " over "), -// new Diff(Operation.DELETE, "the"), -// new Diff(Operation.INSERT, "a"), -// new Diff(Operation.EQUAL, "\nlaz")}; -// string strp = "@@ -21,18 +22,17 @@\n jump\n-s\n+ed\n over \n-the\n+a\n %0alaz\n"; -// assertEquals("Patch: toString.", strp, p.ToString()); -//} +test "patch to text" { + try std.testing.checkAllAllocationFailures( + testing.allocator, + testPatchToText, + .{}, + ); +} + +fn testPatchRoundTrip(allocator: Allocator, patch_in: []const u8) !void { + var patches = try patchFromText(allocator, patch_in); + defer deinitPatchList(allocator, &patches); + const patch_out = try patchToText(allocator, patches); + defer allocator.free(patch_out); + try testing.expectEqualStrings(patch_in, patch_out); +} + +test "patch from text" { + const allocator = testing.allocator; + var p0 = try patchFromText(allocator, ""); + defer deinitPatchList(allocator, &p0); + try testing.expectEqual(0, p0.items.len); + try std.testing.checkAllAllocationFailures( + testing.allocator, + testPatchRoundTrip, + .{"@@ -21,18 +22,17 @@\n jump\n-s\n+ed\n over \n-the\n+a\n %0Alaz\n"}, + ); +} From f07cd608b9484827c6d826a3f68f4ed011f50b9c Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 21:55:39 -0400 Subject: [PATCH 092/176] Patch from text tests --- DiffMatchPatch.zig | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index babf953..0d90fd6 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -3070,7 +3070,6 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc cursor = count; assert(cursor > 4); if (text[cursor] != ',') { - cursor += 1; patch.start1 -= 1; patch.length1 = 1; } else { @@ -3100,13 +3099,12 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc ) catch return error.BadPatchString; cursor += delta1; if (text[cursor] != ',') { - cursor += 1; patch.start2 -= 1; patch.length2 = 1; } else { cursor += 1; const delta2 = countDigits(text[cursor..]); - assert(delta2 > 1); + assert(delta2 > 0); patch.length2 = std.fmt.parseInt( usize, text[cursor .. cursor + delta2], @@ -5112,4 +5110,20 @@ test "patch from text" { testPatchRoundTrip, .{"@@ -21,18 +22,17 @@\n jump\n-s\n+ed\n over \n-the\n+a\n %0Alaz\n"}, ); + try std.testing.checkAllAllocationFailures( + allocator, + testPatchRoundTrip, + .{"@@ -1 +1 @@\n-a\n+b\n"}, + ); + try std.testing.checkAllAllocationFailures( + testing.allocator, + testPatchRoundTrip, + .{"@@ -1,3 +0,0 @@\n-abc\n"}, + ); + try std.testing.checkAllAllocationFailures( + testing.allocator, + testPatchRoundTrip, + .{"@@ -0,0 +1,3 @@\n+abc\n"}, + ); + try testing.expectError(error.BadPatchString, patchFromText(allocator, "Bad\nPatch\nString\n")); } From 689f46fe06fd9135acefc9e50441abfa5de52b40 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 23:41:18 -0400 Subject: [PATCH 093/176] Pass first round of patchAddContext MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The C# code uses a positively cursèd combination of C# Substring, along with what it calls JavaSubstring, which is what sane people use. The Python port got me sorted out. --- DiffMatchPatch.zig | 54 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 0d90fd6..acd675c 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2392,7 +2392,7 @@ fn patchAddContext( // TODO the fixup logic here might make patterns too large? // It should be ok, because big patches get broken up. Hmm. // Also, the SimpleNote maintained branch does it this way. - var padding = 0; + var padding: usize = 0; { // Grow the pattern around the patch until unique, to set padding amount. var pattern = text[patch.start2 .. patch.start2 + patch.length1]; const max_width: usize = dmp.match_max_bits - (2 * dmp.patch_margin); @@ -2413,26 +2413,23 @@ fn patchAddContext( pre_start -= 1; } // Assuming we did everything else right, pre_end should be // properly placed. - const pre_end = pre_start + patch.start2; - break :pre text[pre_start..pre_end]; + break :pre text[pre_start..patch.start2]; }; if (prefix.len != 0) { try patch.diffs.ensureUnusedCapacity(allocator, 1); - patch.diffs.appendAssumeCapacity( - Diff.init( - .equal, - try allocator.dupe(u8, prefix), - ), - ); + patch.diffs.insertAssumeCapacity(0, Diff.init( + .equal, + try allocator.dupe(u8, prefix), + )); } // Add the suffix. const suffix = post: { const post_start = patch.start2 + patch.length1; // In case we messed up somewhere: assert(!is_follow(text[post_start])); - var post_end = post_start + @min(text.len, patch.start2 + patch.length1 + padding); + var post_end = @min(text.len, patch.start2 + patch.length1 + padding); // Prevent broken codepoints here as well: Lead bytes, or follow with another follow - while (!std.ascii.isASCII(text[post_end]) and post_end + 1 < text.len and is_follow(text[post_end + 1])) { + while (post_end + 1 < text.len and !std.ascii.isASCII(text[post_end]) and is_follow(text[post_end + 1])) { post_end += 1; // Special case: penultimate with another follow at end if (post_end + 2 == text.len and is_follow(text[post_end + 1])) { @@ -2440,6 +2437,7 @@ fn patchAddContext( break; // Not actually necessary, but polite. } } + post_end = @min(post_end, text.len); break :post text[post_start..post_end]; }; if (suffix.len != 0) { @@ -5127,3 +5125,37 @@ test "patch from text" { ); try testing.expectError(error.BadPatchString, patchFromText(allocator, "Bad\nPatch\nString\n")); } + +fn testPatchAddContext( + allocator: Allocator, + dmp: DiffMatchPatch, + patch_text: []const u8, + text: []const u8, + expect: []const u8, +) !void { + _, var patch = try patchFromHeader(allocator, patch_text); + defer patch.deinit(allocator); + const patch_og = try patch.asText(allocator); + defer allocator.free(patch_og); + try testing.expectEqualStrings(patch_text, patch_og); + try dmp.patchAddContext(allocator, &patch, text); + const patch_out = try patch.asText(allocator); + defer allocator.free(patch_out); + try testing.expectEqualStrings(expect, patch_out); +} + +test "testPatchAddContext" { + const allocator = testing.allocator; + var dmp = DiffMatchPatch{}; + dmp.patch_margin = 4; + try std.testing.checkAllAllocationFailures( + allocator, + testPatchAddContext, + .{ + dmp, + "@@ -21,4 +21,10 @@\n-jump\n+somersault\n", + "The quick brown fox jumps over the lazy dog.", + "@@ -17,12 +17,18 @@\n fox \n-jump\n+somersault\n s ov\n", + }, + ); +} From 7a51025c2edbeb6c52d02f3dfb1618979e57d18e Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 8 Jul 2024 23:52:52 -0400 Subject: [PATCH 094/176] Test coverage for patchAddContext --- DiffMatchPatch.zig | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index acd675c..9f350fb 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2398,7 +2398,7 @@ fn patchAddContext( const max_width: usize = dmp.match_max_bits - (2 * dmp.patch_margin); while (std.mem.indexOf(u8, text, pattern) != std.mem.lastIndexOf(u8, text, pattern) and pattern.len < max_width) { padding += dmp.patch_margin; - const pat_start = @max(0, patch.start2 - padding); + const pat_start = if (padding > patch.start2) 0 else patch.start2 - padding; const pat_end = pat_start + @min(text.len, patch.start2 + patch.length1 + padding); pattern = text[pat_start..pat_end]; } @@ -2407,7 +2407,7 @@ fn patchAddContext( padding += dmp.patch_margin; // Add the prefix. const prefix = pre: { - var pre_start = @max(0, patch.start2 - padding); + var pre_start = if (padding > patch.start2) 0 else patch.start2 - padding; // Make sure we're not breaking a codepoint. while (is_follow(text[pre_start]) and pre_start > 0) { pre_start -= 1; @@ -5148,6 +5148,7 @@ test "testPatchAddContext" { const allocator = testing.allocator; var dmp = DiffMatchPatch{}; dmp.patch_margin = 4; + // Simple case. try std.testing.checkAllAllocationFailures( allocator, testPatchAddContext, @@ -5158,4 +5159,37 @@ test "testPatchAddContext" { "@@ -17,12 +17,18 @@\n fox \n-jump\n+somersault\n s ov\n", }, ); + // Not enough trailing context. + try std.testing.checkAllAllocationFailures( + allocator, + testPatchAddContext, + .{ + dmp, + "@@ -21,4 +21,10 @@\n-jump\n+somersault\n", + "The quick brown fox jumps.", + "@@ -17,10 +17,16 @@\n fox \n-jump\n+somersault\n s.\n", + }, + ); + // Not enough leading context. + try std.testing.checkAllAllocationFailures( + allocator, + testPatchAddContext, + .{ + dmp, + "@@ -3 +3,2 @@\n-e\n+at\n", + "The quick brown fox jumps.", + "@@ -1,7 +1,8 @@\n Th\n-e\n+at\n qui\n", + }, + ); + // Ambiguity. + try std.testing.checkAllAllocationFailures( + allocator, + testPatchAddContext, + .{ + dmp, + "@@ -3 +3,2 @@\n-e\n+at\n", + "The quick brown fox jumps. The quick brown fox crashes.", + "@@ -1,27 +1,28 @@\n Th\n-e\n+at\n quick brown fox jumps. \n", + }, + ); } From 32fa770a050ada52feb2b9050f1004152781b280 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 11:25:41 -0400 Subject: [PATCH 095/176] Test for makePatch accepts null case --- DiffMatchPatch.zig | 89 ++++++++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 30 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 9f350fb..fca8125 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -122,7 +122,7 @@ pub const Diff = struct { pub const Patch = struct { /// Diffs to be applied - diffs: DiffList, // TODO This should be a Diff + diffs: DiffList = DiffList{}, /// Start of patch in before text start1: usize = 0, length1: usize = 0, @@ -2480,14 +2480,14 @@ fn makePatchInternal( diffs: DiffList, diff_act: DiffHandling, ) !PatchList { - const patches = PatchList{}; + var patches = PatchList{}; if (diffs.items.len == 0) { return patches; // Empty diff means empty patchlist } var patch = Patch{}; - var char_count1 = 0; - var char_count2 = 0; + var char_count1: usize = 0; + var char_count2: usize = 0; // This avoids freeing the original copy of the text: var first_patch = true; var prepatch_text = text; @@ -2498,52 +2498,55 @@ fn makePatchInternal( var postpatch = try std.ArrayList(u8).initCapacity(allocator, text.len); defer postpatch.deinit(); try postpatch.appendSlice(text); - for (diffs.items) |a_diff| { + for (diffs.items, 0..) |a_diff, i| { if (patch.diffs.items.len == 0 and a_diff.operation != .equal) { patch.start1 = char_count1; patch.start2 = char_count2; } switch (a_diff.operation) { .insert => { - const d = if (diff_act == .copy) a_diff.clone(allocator) else a_diff; - try patch.diffs.append(allocator, d); + try patch.diffs.ensureUnusedCapacity(allocator, 1); + const d = if (diff_act == .copy) try a_diff.clone(allocator) else a_diff; + patch.diffs.appendAssumeCapacity(d); patch.length2 += a_diff.text.len; try postpatch.insertSlice(char_count2, a_diff.text); }, .delete => { - // - const d = if (diff_act == .copy) a_diff.clone(allocator) else a_diff; - try patch.diffs.append(allocator, d); + try patch.diffs.ensureUnusedCapacity(allocator, 1); + const d = if (diff_act == .copy) try a_diff.clone(allocator) else a_diff; + patch.diffs.appendAssumeCapacity(d); patch.length1 += a_diff.text.len; - try postpatch.replaceRange(char_count2, a_diff.text.len, .{}); + try postpatch.replaceRange(char_count2, a_diff.text.len, ""); }, .equal => { // - if (a_diff.text.len <= 2 * dmp.patch_margin and patch.diffs.items.len != 0 and a_diff != diffs.items[diffs.items.len]) { + if (a_diff.text.len <= 2 * dmp.patch_margin and patch.diffs.items.len != 0 and !a_diff.eql(diffs.items[diffs.items.len])) { // Small equality inside a patch. - patch.diffs.ensureUnusedCapacity(allocator, 1); + try patch.diffs.ensureUnusedCapacity(allocator, 1); const d = if (diff_act == .copy) try a_diff.clone(allocator) else a_diff; - patch.diffs.appendAssumeCapacity(allocator, d); + patch.diffs.appendAssumeCapacity(d); patch.length1 += a_diff.text.len; patch.length2 += a_diff.text.len; } if (a_diff.text.len >= 2 * dmp.patch_margin) { // Time for a new patch. if (patch.diffs.items.len != 0) { - // free the Diff if we own it + // Free the Diff if we own it. if (diff_act == .own) { allocator.free(a_diff.text); - a_diff.text = ""; // for errdefer + // Replace with null patch to prevent double-free on error + patch.diffs.items[i] = Diff{ .operation = .equal, .text = "" }; } - try patchAddContext(allocator, patch, prepatch_text); - try patches.append(allocator, patch); + try dmp.patchAddContext(allocator, &patch, prepatch_text); + try patches.ensureUnusedCapacity(allocator, 1); + patches.appendAssumeCapacity(patch); patch = Patch{}; // Unlike Unidiff, our patch lists have a rolling context. // https://github.com/google/diff-match-patch/wiki/Unidiff // Update prepatch text & pos to reflect the application of the // just completed patch. if (first_patch) { - // no free on first + // no free on first, we don't own the original text first_patch = false; } else { allocator.free(prepatch_text); @@ -2558,16 +2561,18 @@ fn makePatchInternal( if (a_diff.operation != .insert) { char_count1 += a_diff.text.len; } - if (a_diff.operation != .remove) { + if (a_diff.operation != .delete) { char_count2 += a_diff.text.len; } } // end for loop // Pick up the leftover patch if not empty. if (patch.diffs.items.len != 0) { - try patchAddContext(allocator, patch, prepatch_text); - try patches.append(allocator, patch); + try dmp.patchAddContext(allocator, &patch, prepatch_text); + try patches.ensureUnusedCapacity(allocator, 1); + patches.appendAssumeCapacity(patch); } + return patches; } /// Compute a list of patches to turn text1 into text2. @@ -2584,20 +2589,25 @@ pub fn makePatch( try dmp.makePatchInternal(allocator, text, diffs, .copy); } -pub fn makePatchFromTexts(allocator: Allocator, text1: []const u8, text2: []const u8) !PatchList { - const diffs = try diff(@This(), allocator, text1, text2, true); - errdefer diffs.deinit(allocator); +pub fn makePatchFromTexts( + dmp: DiffMatchPatch, + allocator: Allocator, + text1: []const u8, + text2: []const u8, +) !PatchList { + var diffs = try dmp.diff(allocator, text1, text2, true); + errdefer deinitDiffList(allocator, &diffs); if (diffs.items.len > 2) { - try diffCleanupSemantic(diffs); - try diffCleanupEfficiency(diffs); + try diffCleanupSemantic(allocator, &diffs); + try dmp.diffCleanupEfficiency(allocator, &diffs); } - return try makePatchInternal(allocator, text1, diffs, .own); + return try dmp.makePatchInternal(allocator, text1, diffs, .own); } -pub fn makePatchFromDiffs(allocator: Allocator, diffs: DiffList) !PatchList { +pub fn makePatchFromDiffs(dmp: DiffMatchPatch, allocator: Allocator, diffs: DiffList) !PatchList { const text1 = try diffBeforeText(allocator, diffs); defer allocator.free(text1); - return try makePatch(allocator, text1, diffs, .copy); + return try dmp.makePatch(allocator, text1, diffs, .copy); } /// Merge a set of patches onto the text. Returns a tuple: the first of which @@ -5192,4 +5202,23 @@ test "testPatchAddContext" { "@@ -1,27 +1,28 @@\n Th\n-e\n+at\n quick brown fox jumps. \n", }, ); + // TODO: This will need some patches which check the Unicode handling. +} + +fn testMakePatch(allocator: Allocator) !void { + var dmp = DiffMatchPatch{}; + dmp.match_max_bits = 32; // Need this for compat with translated tests + var null_patch = try dmp.makePatchFromTexts(allocator, "", ""); + defer deinitPatchList(allocator, &null_patch); + const null_patch_text = try patchToText(allocator, null_patch); + defer allocator.free(null_patch_text); + try testing.expectEqualStrings("", null_patch_text); +} + +test "testMakePatch" { + try testing.checkAllAllocationFailures( + testing.allocator, + testMakePatch, + .{}, + ); } From f2a4adbd05e418acf9be2e5bd423db2f7662fbd1 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 12:09:20 -0400 Subject: [PATCH 096/176] Memory leaks inside of diff engine That needs to be fixed on the other branch. --- DiffMatchPatch.zig | 78 +++++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index fca8125..c91abbf 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2472,6 +2472,22 @@ const DiffHandling = enum { own, }; +pub fn makePatchFromTexts( + dmp: DiffMatchPatch, + allocator: Allocator, + text1: []const u8, + text2: []const u8, +) !PatchList { + var diffs = try dmp.diff(allocator, text1, text2, true); + // TODO try this with transfering patches once leaks are sorted out + defer deinitDiffList(allocator, &diffs); + if (diffs.items.len > 2) { + try diffCleanupSemantic(allocator, &diffs); + try dmp.diffCleanupEfficiency(allocator, &diffs); + } // XXX TODO this should use .own once memory issues are sorted out + return try dmp.makePatchInternal(allocator, text1, diffs, .copy); +} + /// @return List of Patch objects. fn makePatchInternal( dmp: DiffMatchPatch, @@ -2481,11 +2497,14 @@ fn makePatchInternal( diff_act: DiffHandling, ) !PatchList { var patches = PatchList{}; + errdefer deinitPatchList(allocator, &patches); if (diffs.items.len == 0) { return patches; // Empty diff means empty patchlist } + _ = diff_act; // XXX figure this out later var patch = Patch{}; + errdefer patch.deinit(allocator); var char_count1: usize = 0; var char_count2: usize = 0; // This avoids freeing the original copy of the text: @@ -2499,6 +2518,7 @@ fn makePatchInternal( defer postpatch.deinit(); try postpatch.appendSlice(text); for (diffs.items, 0..) |a_diff, i| { + _ = i; // XXX if (patch.diffs.items.len == 0 and a_diff.operation != .equal) { patch.start1 = char_count1; patch.start2 = char_count2; @@ -2506,24 +2526,24 @@ fn makePatchInternal( switch (a_diff.operation) { .insert => { try patch.diffs.ensureUnusedCapacity(allocator, 1); - const d = if (diff_act == .copy) try a_diff.clone(allocator) else a_diff; + const d = try a_diff.clone(allocator); // if (diff_act == .copy) try a_diff.clone(allocator) else a_diff; patch.diffs.appendAssumeCapacity(d); patch.length2 += a_diff.text.len; try postpatch.insertSlice(char_count2, a_diff.text); }, .delete => { try patch.diffs.ensureUnusedCapacity(allocator, 1); - const d = if (diff_act == .copy) try a_diff.clone(allocator) else a_diff; + const d = try a_diff.clone(allocator); // if (diff_act == .copy) try a_diff.clone(allocator) else a_diff; patch.diffs.appendAssumeCapacity(d); patch.length1 += a_diff.text.len; try postpatch.replaceRange(char_count2, a_diff.text.len, ""); }, .equal => { // - if (a_diff.text.len <= 2 * dmp.patch_margin and patch.diffs.items.len != 0 and !a_diff.eql(diffs.items[diffs.items.len])) { + if (a_diff.text.len <= 2 * dmp.patch_margin and patch.diffs.items.len != 0 and !a_diff.eql(diffs.getLast())) { // Small equality inside a patch. try patch.diffs.ensureUnusedCapacity(allocator, 1); - const d = if (diff_act == .copy) try a_diff.clone(allocator) else a_diff; + const d = try a_diff.clone(allocator); // if (diff_act == .copy) try a_diff.clone(allocator) else a_diff; patch.diffs.appendAssumeCapacity(d); patch.length1 += a_diff.text.len; patch.length2 += a_diff.text.len; @@ -2532,11 +2552,11 @@ fn makePatchInternal( // Time for a new patch. if (patch.diffs.items.len != 0) { // Free the Diff if we own it. - if (diff_act == .own) { - allocator.free(a_diff.text); - // Replace with null patch to prevent double-free on error - patch.diffs.items[i] = Diff{ .operation = .equal, .text = "" }; - } + // if (diff_act == .own) { + // allocator.free(a_diff.text); + // // Replace with null patch to prevent double-free on error + // diffs.items[i] = Diff{ .operation = .equal, .text = "" }; + // } try dmp.patchAddContext(allocator, &patch, prepatch_text); try patches.ensureUnusedCapacity(allocator, 1); patches.appendAssumeCapacity(patch); @@ -2589,21 +2609,6 @@ pub fn makePatch( try dmp.makePatchInternal(allocator, text, diffs, .copy); } -pub fn makePatchFromTexts( - dmp: DiffMatchPatch, - allocator: Allocator, - text1: []const u8, - text2: []const u8, -) !PatchList { - var diffs = try dmp.diff(allocator, text1, text2, true); - errdefer deinitDiffList(allocator, &diffs); - if (diffs.items.len > 2) { - try diffCleanupSemantic(allocator, &diffs); - try dmp.diffCleanupEfficiency(allocator, &diffs); - } - return try dmp.makePatchInternal(allocator, text1, diffs, .own); -} - pub fn makePatchFromDiffs(dmp: DiffMatchPatch, allocator: Allocator, diffs: DiffList) !PatchList { const text1 = try diffBeforeText(allocator, diffs); defer allocator.free(text1); @@ -5213,12 +5218,27 @@ fn testMakePatch(allocator: Allocator) !void { const null_patch_text = try patchToText(allocator, null_patch); defer allocator.free(null_patch_text); try testing.expectEqualStrings("", null_patch_text); + const text1 = "The quick brown fox jumps over the lazy dog."; + const text2 = "That quick brown fox jumped over a lazy dog."; + const expectedPatch = "@@ -1,8 +1,7 @@\n Th\n-at\n+e\n qui\n@@ -21,17 +21,18 @@\n jump\n-ed\n+s\n over \n-a\n+the\n laz\n"; + // The second patch must be "-21,17 +21,18", not "-22,17 +21,18" due to rolling context. + var diffs = try dmp.diff(allocator, text2, text1, true); + deinitDiffList(allocator, &diffs); + if (false) { + var patches = try dmp.makePatchFromTexts(allocator, text2, text1); + defer deinitPatchList(allocator, &patches); + const patch_text = try patchToText(allocator, patches); + defer allocator.free(patch_text); + try testing.expectEqualStrings(expectedPatch, patch_text); + } } test "testMakePatch" { - try testing.checkAllAllocationFailures( - testing.allocator, - testMakePatch, - .{}, - ); + if (true) { + try testing.checkAllAllocationFailures( + testing.allocator, + testMakePatch, + .{}, + ); + } } From 8a8e49baf4e8128c599bb4c783a7410dca0aae84 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 13:37:40 -0400 Subject: [PATCH 097/176] Manually port memory-management branch changes After the monster patch incident, git has decided to remove everything on this branch if I ever merge in memory-managment. Landing this will be fun. --- DiffMatchPatch.zig | 32 +++++++++---- wtf.patch | 117 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 wtf.patch diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index c91abbf..932550c 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -374,11 +374,10 @@ fn diffCompute( check_lines: bool, deadline: u64, ) DiffError!DiffList { - var diffs = DiffList{}; - errdefer deinitDiffList(allocator, &diffs); - if (before.len == 0) { // Just add some text (speedup). + var diffs = DiffList{}; + errdefer deinitDiffList(allocator, &diffs); try diffs.ensureUnusedCapacity(allocator, 1); diffs.appendAssumeCapacity(Diff.init( .insert, @@ -389,6 +388,8 @@ fn diffCompute( if (after.len == 0) { // Just delete some text (speedup). + var diffs = DiffList{}; + errdefer deinitDiffList(allocator, &diffs); try diffs.ensureUnusedCapacity(allocator, 1); diffs.appendAssumeCapacity(Diff.init( .delete, @@ -402,6 +403,8 @@ fn diffCompute( if (std.mem.indexOf(u8, long_text, short_text)) |index| { // Shorter text is inside the longer text (speedup). + var diffs = DiffList{}; + errdefer deinitDiffList(allocator, &diffs); const op: Diff.Operation = if (before.len > after.len) .delete else @@ -425,6 +428,8 @@ fn diffCompute( if (short_text.len == 1) { // Single character string. // After the previous speedup, the character can't be an equality. + var diffs = DiffList{}; + errdefer deinitDiffList(allocator, &diffs); try diffs.ensureUnusedCapacity(allocator, 2); diffs.appendAssumeCapacity(Diff.init( .delete, @@ -442,13 +447,14 @@ fn diffCompute( // A half-match was found, sort out the return data. defer half_match.deinit(allocator); // Send both pairs off for separate processing. - const diffs_a = try dmp.diffInternal( + var diffs = try dmp.diffInternal( allocator, half_match.prefix_before, half_match.prefix_after, check_lines, deadline, ); + errdefer deinitDiffList(allocator, &diffs); var diffs_b = try dmp.diffInternal( allocator, half_match.suffix_before, @@ -466,7 +472,6 @@ fn diffCompute( } // Merge the results. - diffs = diffs_a; try diffs.ensureUnusedCapacity(allocator, 1); diffs.appendAssumeCapacity( Diff.init(.equal, try allocator.dupe( @@ -628,7 +633,6 @@ fn diffHalfMatchInternal( errdefer allocator.free(prefix_after); const suffix_after = try allocator.dupe(u8, best_short_text_b); const best_common_text = try best_common.toOwnedSlice(allocator); - errdefer allocator.free(best_common_text); return .{ .prefix_before = prefix_before, .suffix_before = suffix_before, @@ -1179,10 +1183,10 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo const ii = pointer - count_delete - count_insert - 1; var nt = try allocator.alloc(u8, diffs.items[ii].text.len + common_length); const ot = diffs.items[ii].text; - defer allocator.free(ot); @memcpy(nt[0..ot.len], ot); @memcpy(nt[ot.len..], text_insert.items[0..common_length]); diffs.items[ii].text = nt; + allocator.free(ot); } else { try diffs.ensureUnusedCapacity(allocator, 1); const text = try allocator.dupe(u8, text_insert.items[0..common_length]); @@ -1201,7 +1205,7 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo text_insert.items[text_insert.items.len - common_length ..], old_text, }); - defer allocator.free(old_text); + allocator.free(old_text); text_insert.items.len -= common_length; text_delete.items.len -= common_length; } @@ -3347,6 +3351,18 @@ fn testDiffHalfMatch( try testing.expectEqualDeep(params.expected, maybe_result); } +fn testDiffHalfMatchLeak(allocator: Allocator) !void { + const dmp = DiffMatchPatch{}; + const text1 = "The quick brown fox jumps over the lazy dog."; + const text2 = "That quick brown fox jumped over a lazy dog."; + var diffs = try dmp.diff(allocator, text2, text1, true); + deinitDiffList(allocator, &diffs); +} + +test "diffHalfMatch leak regression test" { + try testing.checkAllAllocationFailures(testing.allocator, testDiffHalfMatchLeak, .{}); +} + test diffHalfMatch { const one_timeout: DiffMatchPatch = .{ .diff_timeout = 1 }; diff --git a/wtf.patch b/wtf.patch new file mode 100644 index 0000000..0322aff --- /dev/null +++ b/wtf.patch @@ -0,0 +1,117 @@ +diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig +index 2bf574f..dfec56c 100644 +--- a/DiffMatchPatch.zig ++++ b/DiffMatchPatch.zig +@@ -222,11 +222,10 @@ fn diffCompute( + check_lines: bool, + deadline: u64, + ) DiffError!DiffList { +- var diffs = DiffList{}; +- errdefer deinitDiffList(allocator, &diffs); +- + if (before.len == 0) { + // Just add some text (speedup). ++ var diffs = DiffList{}; ++ errdefer deinitDiffList(allocator, &diffs); + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.appendAssumeCapacity(Diff.init( + .insert, +@@ -237,6 +236,8 @@ fn diffCompute( + + if (after.len == 0) { + // Just delete some text (speedup). ++ var diffs = DiffList{}; ++ errdefer deinitDiffList(allocator, &diffs); + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.appendAssumeCapacity(Diff.init( + .delete, +@@ -249,6 +250,8 @@ fn diffCompute( + const short_text = if (before.len > after.len) after else before; + + if (std.mem.indexOf(u8, long_text, short_text)) |index| { ++ var diffs = DiffList{}; ++ errdefer deinitDiffList(allocator, &diffs); + // Shorter text is inside the longer text (speedup). + const op: Diff.Operation = if (before.len > after.len) + .delete +@@ -273,6 +276,8 @@ fn diffCompute( + if (short_text.len == 1) { + // Single character string. + // After the previous speedup, the character can't be an equality. ++ var diffs = DiffList{}; ++ errdefer deinitDiffList(allocator, &diffs); + try diffs.ensureUnusedCapacity(allocator, 2); + diffs.appendAssumeCapacity(Diff.init( + .delete, +@@ -290,13 +295,14 @@ fn diffCompute( + // A half-match was found, sort out the return data. + defer half_match.deinit(allocator); + // Send both pairs off for separate processing. +- const diffs_a = try dmp.diffInternal( ++ var diffs = try dmp.diffInternal( + allocator, + half_match.prefix_before, + half_match.prefix_after, + check_lines, + deadline, + ); ++ errdefer deinitDiffList(allocator, &diffs); + var diffs_b = try dmp.diffInternal( + allocator, + half_match.suffix_before, +@@ -314,7 +320,6 @@ fn diffCompute( + } + + // Merge the results. +- diffs = diffs_a; + try diffs.ensureUnusedCapacity(allocator, 1); + diffs.appendAssumeCapacity( + Diff.init(.equal, try allocator.dupe( +@@ -477,7 +482,6 @@ fn diffHalfMatchInternal( + errdefer allocator.free(prefix_after); + const suffix_after = try allocator.dupe(u8, best_short_text_b); + const best_common_text = try best_common.toOwnedSlice(allocator); +- errdefer allocator.free(best_common_text); + return .{ + .prefix_before = prefix_before, + .suffix_before = suffix_before, +@@ -913,10 +917,10 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo + const ii = pointer - count_delete - count_insert - 1; + var nt = try allocator.alloc(u8, diffs.items[ii].text.len + common_length); + const ot = diffs.items[ii].text; +- defer allocator.free(ot); + @memcpy(nt[0..ot.len], ot); + @memcpy(nt[ot.len..], text_insert.items[0..common_length]); + diffs.items[ii].text = nt; ++ allocator.free(ot); + } else { + try diffs.ensureUnusedCapacity(allocator, 1); + const text = try allocator.dupe(u8, text_insert.items[0..common_length]); +@@ -935,7 +939,7 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo + text_insert.items[text_insert.items.len - common_length ..], + old_text, + }); +- defer allocator.free(old_text); ++ allocator.free(old_text); + text_insert.items.len -= common_length; + text_delete.items.len -= common_length; + } +@@ -2274,6 +2278,18 @@ test diffBisect { + }}); + } + ++fn diffHalfMatchLeak(allocator: Allocator) !void { ++ const dmp = DiffMatchPatch{}; ++ const text1 = "The quick brown fox jumps over the lazy dog."; ++ const text2 = "That quick brown fox jumped over a lazy dog."; ++ var diffs = try dmp.diff(allocator, text2, text1, true); ++ deinitDiffList(allocator, &diffs); ++} ++ ++test "diffHalfMatch leak regression test" { ++ try testing.checkAllAllocationFailures(testing.allocator, diffHalfMatchLeak, .{}); ++} ++ + fn testDiff( + allocator: std.mem.Allocator, + params: struct { From 9a460eceab6e687f2463c29383484c01b61f4fa2 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 14:38:41 -0400 Subject: [PATCH 098/176] First allocating makePatch test passes --- DiffMatchPatch.zig | 74 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 23 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 932550c..a341135 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2483,13 +2483,12 @@ pub fn makePatchFromTexts( text2: []const u8, ) !PatchList { var diffs = try dmp.diff(allocator, text1, text2, true); - // TODO try this with transfering patches once leaks are sorted out defer deinitDiffList(allocator, &diffs); if (diffs.items.len > 2) { try diffCleanupSemantic(allocator, &diffs); try dmp.diffCleanupEfficiency(allocator, &diffs); - } // XXX TODO this should use .own once memory issues are sorted out - return try dmp.makePatchInternal(allocator, text1, diffs, .copy); + } + return try dmp.makePatchInternal(allocator, text1, diffs, .own); } /// @return List of Patch objects. @@ -2505,10 +2504,7 @@ fn makePatchInternal( if (diffs.items.len == 0) { return patches; // Empty diff means empty patchlist } - _ = diff_act; // XXX figure this out later - var patch = Patch{}; - errdefer patch.deinit(allocator); var char_count1: usize = 0; var char_count2: usize = 0; // This avoids freeing the original copy of the text: @@ -2518,11 +2514,13 @@ fn makePatchInternal( if (!first_patch) allocator.free(prepatch_text); } + const dummy_diff = Diff{ .operation = .equal, .text = "" }; var postpatch = try std.ArrayList(u8).initCapacity(allocator, text.len); defer postpatch.deinit(); try postpatch.appendSlice(text); + var patch = Patch{}; for (diffs.items, 0..) |a_diff, i| { - _ = i; // XXX + errdefer patch.deinit(allocator); if (patch.diffs.items.len == 0 and a_diff.operation != .equal) { patch.start1 = char_count1; patch.start2 = char_count2; @@ -2530,14 +2528,32 @@ fn makePatchInternal( switch (a_diff.operation) { .insert => { try patch.diffs.ensureUnusedCapacity(allocator, 1); - const d = try a_diff.clone(allocator); // if (diff_act == .copy) try a_diff.clone(allocator) else a_diff; + const d = the_diff: { + if (diff_act == .copy) { + const new = try a_diff.clone(allocator); + break :the_diff new; + } else { + assert(a_diff.eql(diffs.items[i])); + diffs.items[i] = dummy_diff; + break :the_diff a_diff; + } + }; patch.diffs.appendAssumeCapacity(d); patch.length2 += a_diff.text.len; try postpatch.insertSlice(char_count2, a_diff.text); }, .delete => { try patch.diffs.ensureUnusedCapacity(allocator, 1); - const d = try a_diff.clone(allocator); // if (diff_act == .copy) try a_diff.clone(allocator) else a_diff; + const d = the_diff: { + if (diff_act == .copy) { + const new = try a_diff.clone(allocator); + break :the_diff new; + } else { + assert(a_diff.eql(diffs.items[i])); + diffs.items[i] = dummy_diff; + break :the_diff a_diff; + } + }; patch.diffs.appendAssumeCapacity(d); patch.length1 += a_diff.text.len; try postpatch.replaceRange(char_count2, a_diff.text.len, ""); @@ -2547,7 +2563,16 @@ fn makePatchInternal( if (a_diff.text.len <= 2 * dmp.patch_margin and patch.diffs.items.len != 0 and !a_diff.eql(diffs.getLast())) { // Small equality inside a patch. try patch.diffs.ensureUnusedCapacity(allocator, 1); - const d = try a_diff.clone(allocator); // if (diff_act == .copy) try a_diff.clone(allocator) else a_diff; + const d = the_diff: { + if (diff_act == .copy) { + const new = try a_diff.clone(allocator); + break :the_diff new; + } else { + assert(a_diff.eql(diffs.items[i])); + diffs.items[i] = dummy_diff; + break :the_diff a_diff; + } + }; patch.diffs.appendAssumeCapacity(d); patch.length1 += a_diff.text.len; patch.length2 += a_diff.text.len; @@ -2556,11 +2581,11 @@ fn makePatchInternal( // Time for a new patch. if (patch.diffs.items.len != 0) { // Free the Diff if we own it. - // if (diff_act == .own) { - // allocator.free(a_diff.text); - // // Replace with null patch to prevent double-free on error - // diffs.items[i] = Diff{ .operation = .equal, .text = "" }; - // } + if (diff_act == .own) { + assert(a_diff.eql(diffs.items[i])); + allocator.free(a_diff.text); + diffs.items[i] = dummy_diff; + } try dmp.patchAddContext(allocator, &patch, prepatch_text); try patches.ensureUnusedCapacity(allocator, 1); patches.appendAssumeCapacity(patch); @@ -2569,13 +2594,14 @@ fn makePatchInternal( // https://github.com/google/diff-match-patch/wiki/Unidiff // Update prepatch text & pos to reflect the application of the // just completed patch. + const free_patch_text = prepatch_text; + prepatch_text = try allocator.dupe(u8, postpatch.items); if (first_patch) { // no free on first, we don't own the original text first_patch = false; } else { - allocator.free(prepatch_text); + allocator.free(free_patch_text); } - prepatch_text = try allocator.dupe(u8, postpatch.items); char_count1 = char_count2; } } @@ -2589,7 +2615,7 @@ fn makePatchInternal( char_count2 += a_diff.text.len; } } // end for loop - + errdefer patch.deinit(allocator); // Pick up the leftover patch if not empty. if (patch.diffs.items.len != 0) { try dmp.patchAddContext(allocator, &patch, prepatch_text); @@ -3351,7 +3377,7 @@ fn testDiffHalfMatch( try testing.expectEqualDeep(params.expected, maybe_result); } -fn testDiffHalfMatchLeak(allocator: Allocator) !void { +fn testdiffHalfMatchLeak(allocator: Allocator) !void { const dmp = DiffMatchPatch{}; const text1 = "The quick brown fox jumps over the lazy dog."; const text2 = "That quick brown fox jumped over a lazy dog."; @@ -3360,7 +3386,7 @@ fn testDiffHalfMatchLeak(allocator: Allocator) !void { } test "diffHalfMatch leak regression test" { - try testing.checkAllAllocationFailures(testing.allocator, testDiffHalfMatchLeak, .{}); + try testing.checkAllAllocationFailures(testing.allocator, testdiffHalfMatchLeak, .{}); } test diffHalfMatch { @@ -5238,9 +5264,7 @@ fn testMakePatch(allocator: Allocator) !void { const text2 = "That quick brown fox jumped over a lazy dog."; const expectedPatch = "@@ -1,8 +1,7 @@\n Th\n-at\n+e\n qui\n@@ -21,17 +21,18 @@\n jump\n-ed\n+s\n over \n-a\n+the\n laz\n"; // The second patch must be "-21,17 +21,18", not "-22,17 +21,18" due to rolling context. - var diffs = try dmp.diff(allocator, text2, text1, true); - deinitDiffList(allocator, &diffs); - if (false) { + { var patches = try dmp.makePatchFromTexts(allocator, text2, text1); defer deinitPatchList(allocator, &patches); const patch_text = try patchToText(allocator, patches); @@ -5249,6 +5273,10 @@ fn testMakePatch(allocator: Allocator) !void { } } +test "test testMakePatch" { + try testMakePatch(testing.allocator); +} + test "testMakePatch" { if (true) { try testing.checkAllAllocationFailures( From 0e1377ccb99b4a1a8085e7ab242eab68a718bd57 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 15:15:45 -0400 Subject: [PATCH 099/176] Precalculate maximum space needed for makePatchInternal I want to come up with a better all-around solution for this, the sheer amount of copying going on is galling. But this should give the first allocation enough room to accomodate growing the text, considering that the allocator can reuse the memory, although not in ReleaseSafe or Debug mode. --- DiffMatchPatch.zig | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index a341135..8ddcf89 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2514,10 +2514,25 @@ fn makePatchInternal( if (!first_patch) allocator.free(prepatch_text); } + // Calculate amount of extra bytes needed. + // This should let the allocator reuse freed space. + var extra: isize = 0; + for (diffs.items) |a_diff| { + switch (a_diff.operation) { + .insert => { + extra += @intCast(a_diff.text.len); + }, + .delete => { + extra -= @intCast(a_diff.text.len); + }, + .equal => continue, + } + } + const extra_u: usize = if (extra > 0) @intCast(extra) else 0; const dummy_diff = Diff{ .operation = .equal, .text = "" }; - var postpatch = try std.ArrayList(u8).initCapacity(allocator, text.len); + var postpatch = try std.ArrayList(u8).initCapacity(allocator, text.len + extra_u); defer postpatch.deinit(); - try postpatch.appendSlice(text); + postpatch.appendSliceAssumeCapacity(text); var patch = Patch{}; for (diffs.items, 0..) |a_diff, i| { errdefer patch.deinit(allocator); From 6b7a0f83031db4ff4c59a8abf440a00de995eabb Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 16:26:15 -0400 Subject: [PATCH 100/176] More makePatch --- DiffMatchPatch.zig | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 8ddcf89..1fe76ac 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2651,7 +2651,7 @@ pub fn makePatch( text: []const u8, diffs: DiffList, ) !PatchList { - try dmp.makePatchInternal(allocator, text, diffs, .copy); + return try dmp.makePatchInternal(allocator, text, diffs, .copy); } pub fn makePatchFromDiffs(dmp: DiffMatchPatch, allocator: Allocator, diffs: DiffList) !PatchList { @@ -5277,15 +5277,29 @@ fn testMakePatch(allocator: Allocator) !void { try testing.expectEqualStrings("", null_patch_text); const text1 = "The quick brown fox jumps over the lazy dog."; const text2 = "That quick brown fox jumped over a lazy dog."; - const expectedPatch = "@@ -1,8 +1,7 @@\n Th\n-at\n+e\n qui\n@@ -21,17 +21,18 @@\n jump\n-ed\n+s\n over \n-a\n+the\n laz\n"; - // The second patch must be "-21,17 +21,18", not "-22,17 +21,18" due to rolling context. - { + { // The second patch must be "-21,17 +21,18", not "-22,17 +21,18" due to rolling context. + const expectedPatch = "@@ -1,8 +1,7 @@\n Th\n-at\n+e\n qui\n@@ -21,17 +21,18 @@\n jump\n-ed\n+s\n over \n-a\n+the\n laz\n"; var patches = try dmp.makePatchFromTexts(allocator, text2, text1); defer deinitPatchList(allocator, &patches); const patch_text = try patchToText(allocator, patches); defer allocator.free(patch_text); try testing.expectEqualStrings(expectedPatch, patch_text); } + { + const expectedPatch = "@@ -1,11 +1,12 @@\n Th\n-e\n+at\n quick b\n@@ -22,18 +22,17 @@\n jump\n-s\n+ed\n over \n-the\n+a\n laz\n"; + var patches = try dmp.makePatchFromTexts(allocator, text1, text2); + defer deinitPatchList(allocator, &patches); + const patch_text = try patchToText(allocator, patches); + defer allocator.free(patch_text); + try testing.expectEqualStrings(expectedPatch, patch_text); + var diffs = try dmp.diff(allocator, text1, text2, false); + defer deinitDiffList(allocator, &diffs); + var patches2 = try dmp.makePatch(allocator, text1, diffs); + defer deinitPatchList(allocator, &patches2); + const patch_text_2 = try patchToText(allocator, patches); + defer allocator.free(patch_text_2); + try testing.expectEqualStrings(expectedPatch, patch_text_2); + } } test "test testMakePatch" { From 08fae61c4271385165a184c373b93900ac3390e1 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 17:25:45 -0400 Subject: [PATCH 101/176] Rename to diffAndMakePatch --- DiffMatchPatch.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 1fe76ac..2efac7a 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2476,7 +2476,7 @@ const DiffHandling = enum { own, }; -pub fn makePatchFromTexts( +pub fn diffAndMakePatch( dmp: DiffMatchPatch, allocator: Allocator, text1: []const u8, @@ -5270,7 +5270,7 @@ test "testPatchAddContext" { fn testMakePatch(allocator: Allocator) !void { var dmp = DiffMatchPatch{}; dmp.match_max_bits = 32; // Need this for compat with translated tests - var null_patch = try dmp.makePatchFromTexts(allocator, "", ""); + var null_patch = try dmp.diffAndMakePatch(allocator, "", ""); defer deinitPatchList(allocator, &null_patch); const null_patch_text = try patchToText(allocator, null_patch); defer allocator.free(null_patch_text); @@ -5279,7 +5279,7 @@ fn testMakePatch(allocator: Allocator) !void { const text2 = "That quick brown fox jumped over a lazy dog."; { // The second patch must be "-21,17 +21,18", not "-22,17 +21,18" due to rolling context. const expectedPatch = "@@ -1,8 +1,7 @@\n Th\n-at\n+e\n qui\n@@ -21,17 +21,18 @@\n jump\n-ed\n+s\n over \n-a\n+the\n laz\n"; - var patches = try dmp.makePatchFromTexts(allocator, text2, text1); + var patches = try dmp.diffAndMakePatch(allocator, text2, text1); defer deinitPatchList(allocator, &patches); const patch_text = try patchToText(allocator, patches); defer allocator.free(patch_text); @@ -5287,7 +5287,7 @@ fn testMakePatch(allocator: Allocator) !void { } { const expectedPatch = "@@ -1,11 +1,12 @@\n Th\n-e\n+at\n quick b\n@@ -22,18 +22,17 @@\n jump\n-s\n+ed\n over \n-the\n+a\n laz\n"; - var patches = try dmp.makePatchFromTexts(allocator, text1, text2); + var patches = try dmp.diffAndMakePatch(allocator, text1, text2); defer deinitPatchList(allocator, &patches); const patch_text = try patchToText(allocator, patches); defer allocator.free(patch_text); From 6e414d7b360a9d9fece9f089d0cf7156a03a8e8f Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 18:06:07 -0400 Subject: [PATCH 102/176] Another subscript bug --- DiffMatchPatch.zig | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 2efac7a..ddb9592 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2403,7 +2403,7 @@ fn patchAddContext( while (std.mem.indexOf(u8, text, pattern) != std.mem.lastIndexOf(u8, text, pattern) and pattern.len < max_width) { padding += dmp.patch_margin; const pat_start = if (padding > patch.start2) 0 else patch.start2 - padding; - const pat_end = pat_start + @min(text.len, patch.start2 + patch.length1 + padding); + const pat_end = @min(text.len, patch.start2 + patch.length1 + padding); pattern = text[pat_start..pat_end]; } } @@ -2429,8 +2429,6 @@ fn patchAddContext( // Add the suffix. const suffix = post: { const post_start = patch.start2 + patch.length1; - // In case we messed up somewhere: - assert(!is_follow(text[post_start])); var post_end = @min(text.len, patch.start2 + patch.length1 + padding); // Prevent broken codepoints here as well: Lead bytes, or follow with another follow while (post_end + 1 < text.len and !std.ascii.isASCII(text[post_end]) and is_follow(text[post_end + 1])) { @@ -2657,7 +2655,7 @@ pub fn makePatch( pub fn makePatchFromDiffs(dmp: DiffMatchPatch, allocator: Allocator, diffs: DiffList) !PatchList { const text1 = try diffBeforeText(allocator, diffs); defer allocator.free(text1); - return try dmp.makePatch(allocator, text1, diffs, .copy); + return try dmp.makePatch(allocator, text1, diffs); } /// Merge a set of patches onto the text. Returns a tuple: the first of which @@ -5300,6 +5298,40 @@ fn testMakePatch(allocator: Allocator) !void { defer allocator.free(patch_text_2); try testing.expectEqualStrings(expectedPatch, patch_text_2); } + const expectedPatch2 = "@@ -1,21 +1,21 @@\n-%601234567890-=%5B%5D%5C;',./\n+~!@#$%25%5E&*()_+%7B%7D%7C:%22%3C%3E?\n"; + { + var patches = try dmp.diffAndMakePatch( + allocator, + "`1234567890-=[]\\;',./", + "~!@#$%^&*()_+{}|:\"<>?", + ); + defer deinitPatchList(allocator, &patches); + const patch_text = try patchToText(allocator, patches); + defer allocator.free(patch_text); + try testing.expectEqualStrings(expectedPatch2, patch_text); + } + { + var diffs = try sliceToDiffList(allocator, &.{ + .{ .operation = .delete, .text = "`1234567890-=[]\\;',./" }, + .{ .operation = .insert, .text = "~!@#$%^&*()_+{}|:\"<>?" }, + }); + defer deinitDiffList(allocator, &diffs); + var patches = try dmp.makePatchFromDiffs(allocator, diffs); + defer deinitPatchList(allocator, &patches); + for (patches.items[0].diffs.items, 0..) |a_diff, idx| { + try testing.expect(a_diff.eql(diffs.items[idx])); + } + } + { + const text1a = "abcdef" ** 100; + const text2a = text1a ++ "123"; + const expected_patch = "@@ -573,28 +573,31 @@\n cdefabcdefabcdefabcdefabcdef\n+123\n"; + var patches = try dmp.diffAndMakePatch(allocator, text1a, text2a); + defer deinitPatchList(allocator, &patches); + const patch_text = try patchToText(allocator, patches); + defer allocator.free(patch_text); + try testing.expectEqualStrings(expected_patch, patch_text); + } } test "test testMakePatch" { From eac61d925bb85613cb7ec757db82f2ff0038f94a Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 18:44:04 -0400 Subject: [PATCH 103/176] patchSplitMax compiles (with segfault) About par for the course at this point. It's the gnarliest bit of code, at least, of the stuff I've translated. --- DiffMatchPatch.zig | 105 ++++++++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 40 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index ddb9592..ced623e 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2809,20 +2809,21 @@ pub fn patchApply( fn patchSplitMax( dmp: DiffMatchPatch, allocator: Allocator, - patches: PatchList, -) !PatchList { + patches: *PatchList, +) !void { const patch_size = dmp.match_max_bits; const patch_margin = dmp.patch_margin; - const max_patch_len = patch_size - patch_size - patch_margin; + const max_patch_len = patch_size - patch_margin; // Mutating an array while iterating it? Sure, lets! - var x = 0; - while (x < patches.len) : (x += 1) { - if (patches[x].length1 <= patch_size) continue; + var x_i: isize = 0; + while (x_i < patches.items.len) : (x_i += 1) { + const x: usize = @intCast(x_i); + if (patches.items[x].length1 <= patch_size) continue; // We have a big ol' patch. - const bigpatch = patches.orderedRemove(x); + var bigpatch = patches.orderedRemove(x); defer bigpatch.deinit(allocator); // Prevent incrementing past the next patch: - x -= 1; + x_i -= 1; var start1 = bigpatch.start1; var start2 = bigpatch.start2; // start with an empty precontext so that we can deinit consistently @@ -2831,36 +2832,35 @@ fn patchSplitMax( // Create one of several smaller patches. var patch = Patch{}; var empty = true; - patch.start1 = start1 - precontext.items.len; - patch.start2 = start2 - precontext.items.len; + patch.start1 = start1 - precontext.len; + patch.start2 = start2 - precontext.len; if (precontext.len != 0) { - patch.length2 = precontext.length; + patch.length2 = precontext.len; patch.length1 = patch.length2; try patch.diffs.ensureUnusedCapacity(allocator, 1); patch.diffs.appendAssumeCapacity( - allocator, Diff{ .operation = .equal, - .text = try precontext.toOwnedSlice(), + .text = precontext, }, ); } - while (bigpatch.diffs.count != 0 and patch.length1 < max_patch_len) { - const diff_type = bigpatch.diffs[0].operation; - const diff_text = bigpatch.diffs[0].text; + while (bigpatch.diffs.items.len != 0 and patch.length1 < max_patch_len) { + const diff_type = bigpatch.diffs.items[0].operation; + const diff_text = bigpatch.diffs.items[0].text; if (diff_type == .insert) { // Insertions are harmless. patch.length2 += diff_text.len; start2 += diff_text.len; // Move the patch (transfers ownership) - const diff1 = bigpatch.diffs.orderedRemove(0); - patch.diffs.append(diff1); + try patch.diffs.ensureUnusedCapacity(allocator, 1); + patch.diffs.appendAssumeCapacity(bigpatch.diffs.orderedRemove(0)); empty = false; } else if (cond: { // zig fmt simply will not line break if clauses :/ const a = diff_type == .delete; const b = patch.diffs.items.len == 1; - const c = patch.diffs[0].operation == .equal; + const c = patch.diffs.items[0].operation == .equal; const d = diff_text.len > 2 * patch_size; break :cond a and b and c and d; }) { @@ -2870,13 +2870,12 @@ fn patchSplitMax( empty = false; // Transfer to patch: try patch.diffs.ensureUnusedCapacity(allocator, 1); - const diff1 = bigpatch.diffs.orderedRemove(0); - patch.diffs.appendAssumeCapacity(diff1); + patch.diffs.appendAssumeCapacity(bigpatch.diffs.orderedRemove(0)); } else { // Deletion or equality. Only take as much as we can stomach. const text_end = @min(diff_text.len, patch_size - patch.length1 - patch_margin); const new_diff_text = diff_text[0..text_end]; - patch.length += new_diff_text.len; + patch.length1 += new_diff_text.len; start1 += new_diff_text.len; if (diff_type == .equal) { patch.length2 += diff_text.len; @@ -2888,22 +2887,20 @@ fn patchSplitMax( if (new_diff_text.len == diff_text.len) { // We can reuse the diff. try patch.diffs.ensureUnusedCapacity(allocator, 1); - const diff1 = bigpatch.diffs.orderedRemove(0); - patch.diffs.append(diff1); + patch.diffs.appendAssumeCapacity(bigpatch.diffs.orderedRemove(0)); } else { // Free and dupe - const old_diff = bigpatch.diffs[0]; - errdefer old_diff.deinit(allocator); - bigpatch.diffs[0] = Diff{ + const old_diff = bigpatch.diffs.items[0]; + bigpatch.diffs.items[0] = Diff{ .operation = diff_type, .text = try allocator.dupe(u8, new_diff_text), }; - old_diff.deinit(allocator); + allocator.free(old_diff.text); } } } // Compute the head context for the next patch. - const context_len: isize = precontext.len - patch_margin; + const context_len: usize = if (patch_margin > 0) 0 else precontext.len - patch_margin; allocator.free(precontext); if (context_len > 0) { const after_text = try diffAfterText(allocator, patch.diffs); @@ -2913,7 +2910,7 @@ fn patchSplitMax( precontext = try allocator.alloc(u8, 0); } // Append the end context for this patch. - const post_text = try diffBeforeText(bigpatch.diffs); + const post_text = try diffBeforeText(allocator, bigpatch.diffs); const postcontext = post: { if (post_text.len > patch_margin) { defer allocator.free(post_text); @@ -2929,12 +2926,15 @@ fn patchSplitMax( if (maybe_last_diff) |last_diff| { if (last_diff.operation == .equal) { // free this diff and swap in a new one - defer last_diff.deinit(allocator); + defer allocator.free(last_diff.text); patch.diffs.items.len -= 1; const new_diff_text = try std.mem.concat( allocator, - last_diff.text, - postcontext, + u8, + &.{ + last_diff.text, + postcontext, + }, ); try patch.diffs.append( allocator, @@ -2952,8 +2952,8 @@ fn patchSplitMax( if (!empty) { // Insert the next patch // Goes after x, and we need increment to skip: - x += 1; - try patches.insert(allocator, x, patch); + x_i += 1; + try patches.insert(allocator, @intCast(x_i), patch); } } // Free final precontext. @@ -5334,16 +5334,41 @@ fn testMakePatch(allocator: Allocator) !void { } } -test "test testMakePatch" { - try testMakePatch(testing.allocator); +test makePatch { + try testing.checkAllAllocationFailures( + testing.allocator, + testMakePatch, + .{}, + ); +} + +fn testPatchSplitMax(allocator: Allocator) !void { + var dmp = DiffMatchPatch{}; + // TODO get some tests which cover the max split we actually use: bitsize(usize) + dmp.match_max_bits = 32; + { + var patches = try dmp.diffAndMakePatch( + allocator, + "abcdefghijklmnopqrstuvwxyz01234567890", + "XabXcdXefXghXijXklXmnXopXqrXstXuvXwxXyzX01X23X45X67X89X0", + ); + defer deinitPatchList(allocator, &patches); + const expected_patch = "@@ -1,32 +1,46 @@\n+X\n ab\n+X\n cd\n+X\n ef\n+X\n gh\n+X\n ij\n+X\n kl\n+X\n mn\n+X\n op\n+X\n qr\n+X\n st\n+X\n uv\n+X\n wx\n+X\n yz\n+X\n 012345\n@@ -25,13 +39,18 @@\n zX01\n+X\n 23\n+X\n 45\n+X\n 67\n+X\n 89\n+X\n 0\n"; + try dmp.patchSplitMax(allocator, &patches); + defer deinitPatchList(allocator, &patches); + const patch_text = try patchToText(allocator, patches); + defer allocator.free(patch_text); + try testing.expectEqualStrings(expected_patch, patch_text); + } } -test "testMakePatch" { - if (true) { +test "testPatchSplitMax" { + if (false) { try testing.checkAllAllocationFailures( testing.allocator, - testMakePatch, + testPatchSplitMax, .{}, ); } + try testPatchSplitMax(testing.allocator); } From 1a47673e8cde6b81ab5359aaaa6a4dc973f607fe Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 19:08:49 -0400 Subject: [PATCH 104/176] Less segfaulting But not none. --- DiffMatchPatch.zig | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index ced623e..edb8ed5 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2912,9 +2912,11 @@ fn patchSplitMax( // Append the end context for this patch. const post_text = try diffBeforeText(allocator, bigpatch.diffs); const postcontext = post: { + errdefer allocator.free(post_text); if (post_text.len > patch_margin) { - defer allocator.free(post_text); - break :post post_text[0..patch_margin]; + const truncated = try allocator.dupe(u8, post_text[0..patch_margin]); + allocator.free(post_text); + break :post truncated; } else { break :post post_text; } @@ -2926,8 +2928,12 @@ fn patchSplitMax( if (maybe_last_diff) |last_diff| { if (last_diff.operation == .equal) { // free this diff and swap in a new one - defer allocator.free(last_diff.text); + defer { + allocator.free(last_diff.text); + allocator.free(postcontext); + } patch.diffs.items.len -= 1; + try patch.diffs.ensureUnusedCapacity(allocator, 1); const new_diff_text = try std.mem.concat( allocator, u8, @@ -2936,13 +2942,13 @@ fn patchSplitMax( postcontext, }, ); - try patch.diffs.append( - allocator, + patch.diffs.appendAssumeCapacity( Diff{ .operation = .equal, .text = new_diff_text }, ); } } else { // New diff from postcontext. + errdefer allocator.free(postcontext); try patch.diffs.append( allocator, Diff{ .operation = .equal, .text = postcontext }, From cbb421a9ffd599b67df6d55e5febbb000880b7ad Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 19:10:14 -0400 Subject: [PATCH 105/176] Obvious segfault is obvious --- DiffMatchPatch.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index edb8ed5..d19d334 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -5361,7 +5361,6 @@ fn testPatchSplitMax(allocator: Allocator) !void { defer deinitPatchList(allocator, &patches); const expected_patch = "@@ -1,32 +1,46 @@\n+X\n ab\n+X\n cd\n+X\n ef\n+X\n gh\n+X\n ij\n+X\n kl\n+X\n mn\n+X\n op\n+X\n qr\n+X\n st\n+X\n uv\n+X\n wx\n+X\n yz\n+X\n 012345\n@@ -25,13 +39,18 @@\n zX01\n+X\n 23\n+X\n 45\n+X\n 67\n+X\n 89\n+X\n 0\n"; try dmp.patchSplitMax(allocator, &patches); - defer deinitPatchList(allocator, &patches); const patch_text = try patchToText(allocator, patches); defer allocator.free(patch_text); try testing.expectEqualStrings(expected_patch, patch_text); From 3ae4f985c57afedb190dbf5e57b47293c55230db Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 19:35:33 -0400 Subject: [PATCH 106/176] Correct answer for first splitMaxPatch test Not yet subjected to the Testing of The Allocation Failures. --- DiffMatchPatch.zig | 21 ++++++++++----------- a.txt | 41 +++++++++++++++++++++++++++++++++++++++++ b.txt | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 11 deletions(-) create mode 100644 a.txt create mode 100644 b.txt diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index d19d334..7eae30f 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2827,7 +2827,7 @@ fn patchSplitMax( var start1 = bigpatch.start1; var start2 = bigpatch.start2; // start with an empty precontext so that we can deinit consistently - var precontext = try allocator.alloc(u8, 0); + var precontext: []const u8 = try allocator.alloc(u8, 0); while (bigpatch.diffs.items.len != 0) { // Create one of several smaller patches. var patch = Patch{}; @@ -2899,15 +2899,15 @@ fn patchSplitMax( } } } - // Compute the head context for the next patch. - const context_len: usize = if (patch_margin > 0) 0 else precontext.len - patch_margin; - allocator.free(precontext); - if (context_len > 0) { - const after_text = try diffAfterText(allocator, patch.diffs); - defer allocator.free(after_text); - precontext = try allocator.dupe(u8, after_text[context_len..]); + // Compute the head context for the next patch + // TODO we don't use the last of these, so we can detect that + // condition and not creat it to begin with. + const after_text = try diffAfterText(allocator, patch.diffs); + if (patch_margin > after_text.len) { + precontext = after_text; } else { - precontext = try allocator.alloc(u8, 0); + defer allocator.free(after_text); + precontext = try allocator.dupe(u8, after_text[after_text.len - patch_margin ..]); } // Append the end context for this patch. const post_text = try diffBeforeText(allocator, bigpatch.diffs); @@ -2961,8 +2961,7 @@ fn patchSplitMax( x_i += 1; try patches.insert(allocator, @intCast(x_i), patch); } - } - // Free final precontext. + } // We don't use the last precontext allocator.free(precontext); } } diff --git a/a.txt b/a.txt new file mode 100644 index 0000000..7e3b317 --- /dev/null +++ b/a.txt @@ -0,0 +1,41 @@ +@@ -1,32 +1,46 @@ ++X + ab ++X + cd ++X + ef ++X + gh ++X + ij ++X + kl ++X + mn ++X + op ++X + qr ++X + st ++X + uv ++X + wx ++X + yz ++X + 012345 +@@ -25,13 +39,18 @@ + zX01 ++X + 23 ++X + 45 ++X + 67 ++X + 89 ++X + 0 \ No newline at end of file diff --git a/b.txt b/b.txt new file mode 100644 index 0000000..fc3e3a7 --- /dev/null +++ b/b.txt @@ -0,0 +1,40 @@ +@@ -1,32 +1,46 @@ ++X + ab ++X + cd ++X + ef ++X + gh ++X + ij ++X + kl ++X + mn ++X + op ++X + qr ++X + st ++X + uv ++X + wx ++X + yz ++X + 012345 +@@ -29,9 +43,14 @@ ++X + 23 ++X + 45 ++X + 67 ++X + 89 ++X + 0 \ No newline at end of file From fa906fafa8c9063b6b082151b35176dc8d1f063f Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 19:48:24 -0400 Subject: [PATCH 107/176] Remove spurious files --- a.txt | 41 ------------------- b.txt | 40 ------------------- wtf.patch | 117 ------------------------------------------------------ 3 files changed, 198 deletions(-) delete mode 100644 a.txt delete mode 100644 b.txt delete mode 100644 wtf.patch diff --git a/a.txt b/a.txt deleted file mode 100644 index 7e3b317..0000000 --- a/a.txt +++ /dev/null @@ -1,41 +0,0 @@ -@@ -1,32 +1,46 @@ -+X - ab -+X - cd -+X - ef -+X - gh -+X - ij -+X - kl -+X - mn -+X - op -+X - qr -+X - st -+X - uv -+X - wx -+X - yz -+X - 012345 -@@ -25,13 +39,18 @@ - zX01 -+X - 23 -+X - 45 -+X - 67 -+X - 89 -+X - 0 \ No newline at end of file diff --git a/b.txt b/b.txt deleted file mode 100644 index fc3e3a7..0000000 --- a/b.txt +++ /dev/null @@ -1,40 +0,0 @@ -@@ -1,32 +1,46 @@ -+X - ab -+X - cd -+X - ef -+X - gh -+X - ij -+X - kl -+X - mn -+X - op -+X - qr -+X - st -+X - uv -+X - wx -+X - yz -+X - 012345 -@@ -29,9 +43,14 @@ -+X - 23 -+X - 45 -+X - 67 -+X - 89 -+X - 0 \ No newline at end of file diff --git a/wtf.patch b/wtf.patch deleted file mode 100644 index 0322aff..0000000 --- a/wtf.patch +++ /dev/null @@ -1,117 +0,0 @@ -diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig -index 2bf574f..dfec56c 100644 ---- a/DiffMatchPatch.zig -+++ b/DiffMatchPatch.zig -@@ -222,11 +222,10 @@ fn diffCompute( - check_lines: bool, - deadline: u64, - ) DiffError!DiffList { -- var diffs = DiffList{}; -- errdefer deinitDiffList(allocator, &diffs); -- - if (before.len == 0) { - // Just add some text (speedup). -+ var diffs = DiffList{}; -+ errdefer deinitDiffList(allocator, &diffs); - try diffs.ensureUnusedCapacity(allocator, 1); - diffs.appendAssumeCapacity(Diff.init( - .insert, -@@ -237,6 +236,8 @@ fn diffCompute( - - if (after.len == 0) { - // Just delete some text (speedup). -+ var diffs = DiffList{}; -+ errdefer deinitDiffList(allocator, &diffs); - try diffs.ensureUnusedCapacity(allocator, 1); - diffs.appendAssumeCapacity(Diff.init( - .delete, -@@ -249,6 +250,8 @@ fn diffCompute( - const short_text = if (before.len > after.len) after else before; - - if (std.mem.indexOf(u8, long_text, short_text)) |index| { -+ var diffs = DiffList{}; -+ errdefer deinitDiffList(allocator, &diffs); - // Shorter text is inside the longer text (speedup). - const op: Diff.Operation = if (before.len > after.len) - .delete -@@ -273,6 +276,8 @@ fn diffCompute( - if (short_text.len == 1) { - // Single character string. - // After the previous speedup, the character can't be an equality. -+ var diffs = DiffList{}; -+ errdefer deinitDiffList(allocator, &diffs); - try diffs.ensureUnusedCapacity(allocator, 2); - diffs.appendAssumeCapacity(Diff.init( - .delete, -@@ -290,13 +295,14 @@ fn diffCompute( - // A half-match was found, sort out the return data. - defer half_match.deinit(allocator); - // Send both pairs off for separate processing. -- const diffs_a = try dmp.diffInternal( -+ var diffs = try dmp.diffInternal( - allocator, - half_match.prefix_before, - half_match.prefix_after, - check_lines, - deadline, - ); -+ errdefer deinitDiffList(allocator, &diffs); - var diffs_b = try dmp.diffInternal( - allocator, - half_match.suffix_before, -@@ -314,7 +320,6 @@ fn diffCompute( - } - - // Merge the results. -- diffs = diffs_a; - try diffs.ensureUnusedCapacity(allocator, 1); - diffs.appendAssumeCapacity( - Diff.init(.equal, try allocator.dupe( -@@ -477,7 +482,6 @@ fn diffHalfMatchInternal( - errdefer allocator.free(prefix_after); - const suffix_after = try allocator.dupe(u8, best_short_text_b); - const best_common_text = try best_common.toOwnedSlice(allocator); -- errdefer allocator.free(best_common_text); - return .{ - .prefix_before = prefix_before, - .suffix_before = suffix_before, -@@ -913,10 +917,10 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo - const ii = pointer - count_delete - count_insert - 1; - var nt = try allocator.alloc(u8, diffs.items[ii].text.len + common_length); - const ot = diffs.items[ii].text; -- defer allocator.free(ot); - @memcpy(nt[0..ot.len], ot); - @memcpy(nt[ot.len..], text_insert.items[0..common_length]); - diffs.items[ii].text = nt; -+ allocator.free(ot); - } else { - try diffs.ensureUnusedCapacity(allocator, 1); - const text = try allocator.dupe(u8, text_insert.items[0..common_length]); -@@ -935,7 +939,7 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo - text_insert.items[text_insert.items.len - common_length ..], - old_text, - }); -- defer allocator.free(old_text); -+ allocator.free(old_text); - text_insert.items.len -= common_length; - text_delete.items.len -= common_length; - } -@@ -2274,6 +2278,18 @@ test diffBisect { - }}); - } - -+fn diffHalfMatchLeak(allocator: Allocator) !void { -+ const dmp = DiffMatchPatch{}; -+ const text1 = "The quick brown fox jumps over the lazy dog."; -+ const text2 = "That quick brown fox jumped over a lazy dog."; -+ var diffs = try dmp.diff(allocator, text2, text1, true); -+ deinitDiffList(allocator, &diffs); -+} -+ -+test "diffHalfMatch leak regression test" { -+ try testing.checkAllAllocationFailures(testing.allocator, diffHalfMatchLeak, .{}); -+} -+ - fn testDiff( - allocator: std.mem.Allocator, - params: struct { From f97e299707aa9a78c50b0bb21de60d86d4e44ce7 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 21:52:05 -0400 Subject: [PATCH 108/176] This algorithm sucks It's a bunch of promiscuous mutation that only happens because of the limitations of a fuzzy search algorithm that I don't even care about. It sucks. No one who cares about performance or resource management would write this to begin with. Ugh. --- DiffMatchPatch.zig | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 7eae30f..318b8da 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2819,6 +2819,7 @@ fn patchSplitMax( while (x_i < patches.items.len) : (x_i += 1) { const x: usize = @intCast(x_i); if (patches.items[x].length1 <= patch_size) continue; + // We have a big ol' patch. var bigpatch = patches.orderedRemove(x); defer bigpatch.deinit(allocator); @@ -2960,6 +2961,8 @@ fn patchSplitMax( // Goes after x, and we need increment to skip: x_i += 1; try patches.insert(allocator, @intCast(x_i), patch); + } else { + patch.deinit(allocator); } } // We don't use the last precontext allocator.free(precontext); @@ -5364,6 +5367,20 @@ fn testPatchSplitMax(allocator: Allocator) !void { defer allocator.free(patch_text); try testing.expectEqualStrings(expected_patch, patch_text); } + { + var patches = try dmp.diffAndMakePatch( + allocator, + "abcdef1234567890123456789012345678901234567890123456789012345678901234567890uvwxyz", + "abcdefuvwxyz", + ); + defer deinitPatchList(allocator, &patches); + const text_before = try patchToText(allocator, patches); + defer allocator.free(text_before); + try dmp.patchSplitMax(allocator, &patches); + const text_after = try patchToText(allocator, patches); + defer allocator.free(text_after); + try testing.expectEqualStrings(text_before, text_after); + } } test "testPatchSplitMax" { From be264213a0b12c12e76f8ed7cda3236fc0770820 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 22:30:24 -0400 Subject: [PATCH 109/176] No allocation failures for first splitMax tests The algorithm still sucks, but I think I've picked most of the hair off it. --- DiffMatchPatch.zig | 77 +++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 318b8da..24cd923 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2830,8 +2830,15 @@ fn patchSplitMax( // start with an empty precontext so that we can deinit consistently var precontext: []const u8 = try allocator.alloc(u8, 0); while (bigpatch.diffs.items.len != 0) { + var guard_precontext = true; + errdefer { + if (guard_precontext) { + allocator.free(precontext); + } + } // Create one of several smaller patches. var patch = Patch{}; + errdefer patch.deinit(allocator); var empty = true; patch.start1 = start1 - precontext.len; patch.start2 = start2 - precontext.len; @@ -2839,6 +2846,7 @@ fn patchSplitMax( patch.length2 = precontext.len; patch.length1 = patch.length2; try patch.diffs.ensureUnusedCapacity(allocator, 1); + guard_precontext = false; patch.diffs.appendAssumeCapacity( Diff{ .operation = .equal, @@ -2857,13 +2865,12 @@ fn patchSplitMax( try patch.diffs.ensureUnusedCapacity(allocator, 1); patch.diffs.appendAssumeCapacity(bigpatch.diffs.orderedRemove(0)); empty = false; - } else if (cond: { + } else if (patch.diffs.items.len == 1 and cond: { // zig fmt simply will not line break if clauses :/ const a = diff_type == .delete; - const b = patch.diffs.items.len == 1; - const c = patch.diffs.items[0].operation == .equal; - const d = diff_text.len > 2 * patch_size; - break :cond a and b and c and d; + const b = patch.diffs.items[0].operation == .equal; + const c = diff_text.len > 2 * patch_size; + break :cond a and b and c; }) { // This is a large deletion. Let it pass in one chunk. patch.length1 += diff_text.len; @@ -2910,48 +2917,44 @@ fn patchSplitMax( defer allocator.free(after_text); precontext = try allocator.dupe(u8, after_text[after_text.len - patch_margin ..]); } + guard_precontext = true; // Append the end context for this patch. const post_text = try diffBeforeText(allocator, bigpatch.diffs); const postcontext = post: { - errdefer allocator.free(post_text); if (post_text.len > patch_margin) { + defer allocator.free(post_text); const truncated = try allocator.dupe(u8, post_text[0..patch_margin]); - allocator.free(post_text); break :post truncated; } else { break :post post_text; } }; if (postcontext.len != 0) { + try patch.diffs.ensureUnusedCapacity(allocator, 1); patch.length1 += postcontext.len; patch.length2 += postcontext.len; - const maybe_last_diff = patch.diffs.getLastOrNull(); - if (maybe_last_diff) |last_diff| { - if (last_diff.operation == .equal) { - // free this diff and swap in a new one - defer { - allocator.free(last_diff.text); - allocator.free(postcontext); - } - patch.diffs.items.len -= 1; - try patch.diffs.ensureUnusedCapacity(allocator, 1); - const new_diff_text = try std.mem.concat( - allocator, - u8, - &.{ - last_diff.text, - postcontext, - }, - ); - patch.diffs.appendAssumeCapacity( - Diff{ .operation = .equal, .text = new_diff_text }, - ); + const last_diff = patch.diffs.getLastOrNull(); + if (last_diff != null and last_diff.?.operation == .equal) { + // free this diff and swap in a new one + defer { + allocator.free(last_diff.?.text); + allocator.free(postcontext); } + patch.diffs.items.len -= 1; + const new_diff_text = try std.mem.concat( + allocator, + u8, + &.{ + last_diff.?.text, + postcontext, + }, + ); + patch.diffs.appendAssumeCapacity( + Diff{ .operation = .equal, .text = new_diff_text }, + ); } else { // New diff from postcontext. - errdefer allocator.free(postcontext); - try patch.diffs.append( - allocator, + patch.diffs.appendAssumeCapacity( Diff{ .operation = .equal, .text = postcontext }, ); } @@ -5384,12 +5387,10 @@ fn testPatchSplitMax(allocator: Allocator) !void { } test "testPatchSplitMax" { - if (false) { - try testing.checkAllAllocationFailures( - testing.allocator, - testPatchSplitMax, - .{}, - ); - } + try testing.checkAllAllocationFailures( + testing.allocator, + testPatchSplitMax, + .{}, + ); try testPatchSplitMax(testing.allocator); } From d18e74319785d513c75b04d07c95c9208af1a7ce Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Tue, 9 Jul 2024 23:32:57 -0400 Subject: [PATCH 110/176] Last patchSplitMax test Not convinced I don't need several more of those, in fact, I'm quite suure I do. But the suite, it passes. Much like my bygone youth. --- DiffMatchPatch.zig | 84 +++++++++++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 23 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 24cd923..8de38aa 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2844,7 +2844,7 @@ fn patchSplitMax( patch.start2 = start2 - precontext.len; if (precontext.len != 0) { patch.length2 = precontext.len; - patch.length1 = patch.length2; + patch.length1 = precontext.len; try patch.diffs.ensureUnusedCapacity(allocator, 1); guard_precontext = false; patch.diffs.appendAssumeCapacity( @@ -2886,22 +2886,26 @@ fn patchSplitMax( patch.length1 += new_diff_text.len; start1 += new_diff_text.len; if (diff_type == .equal) { - patch.length2 += diff_text.len; - start2 += diff_text.len; + patch.length2 += new_diff_text.len; + start2 += new_diff_text.len; } else { empty = false; } // Now check if we did anything. + try patch.diffs.ensureUnusedCapacity(allocator, 1); if (new_diff_text.len == diff_text.len) { // We can reuse the diff. - try patch.diffs.ensureUnusedCapacity(allocator, 1); patch.diffs.appendAssumeCapacity(bigpatch.diffs.orderedRemove(0)); } else { // Free and dupe + patch.diffs.appendAssumeCapacity(Diff{ + .operation = diff_type, + .text = try allocator.dupe(u8, new_diff_text), + }); const old_diff = bigpatch.diffs.items[0]; bigpatch.diffs.items[0] = Diff{ .operation = diff_type, - .text = try allocator.dupe(u8, new_diff_text), + .text = try allocator.dupe(u8, diff_text[new_diff_text.len..]), }; allocator.free(old_diff.text); } @@ -2935,7 +2939,7 @@ fn patchSplitMax( patch.length2 += postcontext.len; const last_diff = patch.diffs.getLastOrNull(); if (last_diff != null and last_diff.?.operation == .equal) { - // free this diff and swap in a new one + // Free this diff and swap in a new one. defer { allocator.free(last_diff.?.text); allocator.free(postcontext); @@ -5357,32 +5361,66 @@ fn testPatchSplitMax(allocator: Allocator) !void { var dmp = DiffMatchPatch{}; // TODO get some tests which cover the max split we actually use: bitsize(usize) dmp.match_max_bits = 32; + if (false) { + { + var patches = try dmp.diffAndMakePatch( + allocator, + "abcdefghijklmnopqrstuvwxyz01234567890", + "XabXcdXefXghXijXklXmnXopXqrXstXuvXwxXyzX01X23X45X67X89X0", + ); + defer deinitPatchList(allocator, &patches); + const expected_patch = "@@ -1,32 +1,46 @@\n+X\n ab\n+X\n cd\n+X\n ef\n+X\n gh\n+X\n ij\n+X\n kl\n+X\n mn\n+X\n op\n+X\n qr\n+X\n st\n+X\n uv\n+X\n wx\n+X\n yz\n+X\n 012345\n@@ -25,13 +39,18 @@\n zX01\n+X\n 23\n+X\n 45\n+X\n 67\n+X\n 89\n+X\n 0\n"; + try dmp.patchSplitMax(allocator, &patches); + const patch_text = try patchToText(allocator, patches); + defer allocator.free(patch_text); + try testing.expectEqualStrings(expected_patch, patch_text); + } + { + var patches = try dmp.diffAndMakePatch( + allocator, + "abcdef1234567890123456789012345678901234567890123456789012345678901234567890uvwxyz", + "abcdefuvwxyz", + ); + defer deinitPatchList(allocator, &patches); + const text_before = try patchToText(allocator, patches); + defer allocator.free(text_before); + try dmp.patchSplitMax(allocator, &patches); + const text_after = try patchToText(allocator, patches); + defer allocator.free(text_after); + try testing.expectEqualStrings(text_before, text_after); + } + { + var patches = try dmp.diffAndMakePatch( + allocator, + "1234567890123456789012345678901234567890123456789012345678901234567890", + "abc", + ); + defer deinitPatchList(allocator, &patches); + const pre_patch_text = try patchToText(allocator, patches); + defer allocator.free(pre_patch_text); + try dmp.patchSplitMax(allocator, &patches); + const patch_text = try patchToText(allocator, patches); + defer allocator.free(patch_text); + try testing.expectEqualStrings( + "@@ -1,32 +1,4 @@\n-1234567890123456789012345678\n 9012\n@@ -29,32 +1,4 @@\n-9012345678901234567890123456\n 7890\n@@ -57,14 +1,3 @@\n-78901234567890\n+abc\n", + patch_text, + ); + } + } { var patches = try dmp.diffAndMakePatch( allocator, - "abcdefghijklmnopqrstuvwxyz01234567890", - "XabXcdXefXghXijXklXmnXopXqrXstXuvXwxXyzX01X23X45X67X89X0", + "abcdefghij , h : 0 , t : 1 abcdefghij , h : 0 , t : 1 abcdefghij , h : 0 , t : 1", + "abcdefghij , h : 1 , t : 1 abcdefghij , h : 1 , t : 1 abcdefghij , h : 0 , t : 1", ); defer deinitPatchList(allocator, &patches); - const expected_patch = "@@ -1,32 +1,46 @@\n+X\n ab\n+X\n cd\n+X\n ef\n+X\n gh\n+X\n ij\n+X\n kl\n+X\n mn\n+X\n op\n+X\n qr\n+X\n st\n+X\n uv\n+X\n wx\n+X\n yz\n+X\n 012345\n@@ -25,13 +39,18 @@\n zX01\n+X\n 23\n+X\n 45\n+X\n 67\n+X\n 89\n+X\n 0\n"; try dmp.patchSplitMax(allocator, &patches); const patch_text = try patchToText(allocator, patches); defer allocator.free(patch_text); - try testing.expectEqualStrings(expected_patch, patch_text); - } - { - var patches = try dmp.diffAndMakePatch( - allocator, - "abcdef1234567890123456789012345678901234567890123456789012345678901234567890uvwxyz", - "abcdefuvwxyz", + try testing.expectEqualStrings( + "@@ -2,32 +2,32 @@\n bcdefghij , h : \n-0\n+1\n , t : 1 abcdef\n@@ -29,32 +29,32 @@\n bcdefghij , h : \n-0\n+1\n , t : 1 abcdef\n", + patch_text, ); - defer deinitPatchList(allocator, &patches); - const text_before = try patchToText(allocator, patches); - defer allocator.free(text_before); - try dmp.patchSplitMax(allocator, &patches); - const text_after = try patchToText(allocator, patches); - defer allocator.free(text_after); - try testing.expectEqualStrings(text_before, text_after); } } From 30d203a1056cb35090156d6a582792d9ea647f3e Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 10:27:43 -0400 Subject: [PATCH 111/176] patchAddPadding compiles --- DiffMatchPatch.zig | 89 +++++++++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 29 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 8de38aa..d70aa71 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2046,8 +2046,7 @@ pub fn matchMain( // Clamp the loc to fit within text. const loc = @min(passed_loc, text.len); if (std.mem.eql(u8, text, pattern)) { - // Shortcut (potentially not guaranteed by the algorithm) - // TODO would be good to know what the above means... + // Shortcut return 0; } else if (text.len == 0) { // Nothing to match. @@ -2881,6 +2880,9 @@ fn patchSplitMax( patch.diffs.appendAssumeCapacity(bigpatch.diffs.orderedRemove(0)); } else { // Deletion or equality. Only take as much as we can stomach. + // Note: because this is an internal function, we don't care + // about codepoint splitting, which won't affect the final + // result. const text_end = @min(diff_text.len, patch_size - patch.length1 - patch_margin); const new_diff_text = diff_text[0..text_end]; patch.length1 += new_diff_text.len; @@ -2911,17 +2913,6 @@ fn patchSplitMax( } } } - // Compute the head context for the next patch - // TODO we don't use the last of these, so we can detect that - // condition and not creat it to begin with. - const after_text = try diffAfterText(allocator, patch.diffs); - if (patch_margin > after_text.len) { - precontext = after_text; - } else { - defer allocator.free(after_text); - precontext = try allocator.dupe(u8, after_text[after_text.len - patch_margin ..]); - } - guard_precontext = true; // Append the end context for this patch. const post_text = try diffBeforeText(allocator, bigpatch.diffs); const postcontext = post: { @@ -2933,6 +2924,18 @@ fn patchSplitMax( break :post post_text; } }; + // Compute the head context for the next patch, if we're going to + // need it. + if (bigpatch.diffs.items.len != 0) { + const after_text = try diffAfterText(allocator, patch.diffs); + if (patch_margin > after_text.len) { + precontext = after_text; + } else { + defer allocator.free(after_text); + precontext = try allocator.dupe(u8, after_text[after_text.len - patch_margin ..]); + } + guard_precontext = true; + } if (postcontext.len != 0) { try patch.diffs.ensureUnusedCapacity(allocator, 1); patch.length1 += postcontext.len; @@ -2972,7 +2975,7 @@ fn patchSplitMax( patch.deinit(allocator); } } // We don't use the last precontext - allocator.free(precontext); + // allocator.free(precontext); } } @@ -2983,22 +2986,23 @@ fn patchSplitMax( fn patchAddPadding( dmp: DiffMatchPatch, allocator: Allocator, - patches: PatchList, + patches: *PatchList, ) ![]const u8 { assert(patches.items.len != 0); const pad_len = dmp.patch_margin; var paddingcodes = try std.ArrayList(u8).initCapacity(allocator, pad_len); defer paddingcodes.deinit(); + { var control_code: u8 = 1; while (control_code <= pad_len) : (control_code += 1) { - try paddingcodes.append(control_code); + paddingcodes.appendAssumeCapacity(control_code); } } // Bump all the patches forward. - for (patches) |a_patch| { - a_patch.start1 += pad_len; - a_patch.start2 += pad_len; + for (patches.items) |*a_patch| { + a_patch.*.start1 += pad_len; + a_patch.*.start2 += pad_len; } // Add some padding on start of first diff. var patch = patches.items[0]; @@ -3006,7 +3010,7 @@ fn patchAddPadding( if (diffs.items.len == 0 or diffs.items[0].operation != .equal) { // Add nullPadding equality. try diffs.ensureUnusedCapacity(allocator, 1); - diffs.insert( + diffs.insertAssumeCapacity( 0, Diff{ .operation = .equal, @@ -3019,7 +3023,7 @@ fn patchAddPadding( patch.start2 -= pad_len; assert(patch.start2 == 0); patch.length1 += pad_len; - patch.lenght2 += pad_len; + patch.length2 += pad_len; } else if (pad_len > diffs.items[0].text.len) { // Grow first equality. var diff1 = diffs.items[0]; @@ -3027,8 +3031,8 @@ fn patchAddPadding( const extra_len = pad_len - diff1.text.len; diff1.text = try std.mem.concat( allocator, - paddingcodes.items[diff1.text.len..], - diff1.text, + u8, + &.{ paddingcodes.items[diff1.text.len..], diff1.text }, ); patch.start1 -= extra_len; patch.start2 -= extra_len; @@ -3038,7 +3042,7 @@ fn patchAddPadding( // Add some padding on end of last diff. patch = patches.getLast(); diffs = patch.diffs; - if (diffs.items.len == 0 or diffs.getLast().opeation != .equal) { + if (diffs.items.len == 0 or diffs.getLast().operation != .equal) { // Add nullPadding equality. try diffs.ensureUnusedCapacity(allocator, 1); diffs.appendAssumeCapacity( @@ -3056,8 +3060,8 @@ fn patchAddPadding( const extra_len = pad_len - last_diff.text.len; last_diff.text = try std.mem.concat( allocator, - last_diff.text, - paddingcodes[0..extra_len], + u8, + &.{ last_diff.text, paddingcodes.items[0..extra_len] }, ); patch.length1 += extra_len; patch.length2 += extra_len; @@ -4118,7 +4122,8 @@ test diffBisect { .dmp = this, .before = a, .after = b, - .deadline = std.math.maxInt(u64), // Travis TODO not sure if maxInt(u64) is correct for DateTime.MaxValue + // std.time returns an i64 + .deadline = std.math.maxInt(i64), .expected = &.{ .{ .operation = .delete, .text = "c" }, .{ .operation = .insert, .text = "m" }, @@ -4133,7 +4138,7 @@ test diffBisect { .dmp = this, .before = a, .after = b, - .deadline = 0, // Travis TODO not sure if 0 is correct for DateTime.MinValue + .deadline = 0, // Do not run prior to 1970 .expected = &.{ .{ .operation = .delete, .text = "cat" }, .{ .operation = .insert, .text = "map" }, @@ -5424,7 +5429,7 @@ fn testPatchSplitMax(allocator: Allocator) !void { } } -test "testPatchSplitMax" { +test patchSplitMax { try testing.checkAllAllocationFailures( testing.allocator, testPatchSplitMax, @@ -5432,3 +5437,29 @@ test "testPatchSplitMax" { ); try testPatchSplitMax(testing.allocator); } + +fn testPatchAddPadding( + allocator: Allocator, + params: struct { []const u8, []const u8, []const u8 }, +) !void { + const dmp = DiffMatchPatch{}; + const before, const after, const expect = params; + var patches = try dmp.diffAndMakePatch(allocator, before, after); + defer deinitPatchList(allocator, &patches); + const codes = try dmp.patchAddPadding(allocator, &patches); + allocator.free(codes); + const patch_text = try patchToText(allocator, patches); + defer allocator.free(patch_text); + try testing.expectEqualStrings(expect, patch_text); +} + +test "patchAddPadding" { + try testPatchAddPadding( + testing.allocator, + .{ + "", + "test", + "@@ -0,0 +1,4 @@\n+test\n", + }, + ); +} From d1e1b5830f30f62597833f31a71a418d142d833b Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 12:39:05 -0400 Subject: [PATCH 112/176] Take slices by pointer That way, when .len gets mutated, it shows up in the original patch structure. --- DiffMatchPatch.zig | 86 ++++++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 38 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index d70aa71..0b4ba5e 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2988,7 +2988,7 @@ fn patchAddPadding( allocator: Allocator, patches: *PatchList, ) ![]const u8 { - assert(patches.items.len != 0); + if (patches.items.len == 0) return ""; const pad_len = dmp.patch_margin; var paddingcodes = try std.ArrayList(u8).initCapacity(allocator, pad_len); defer paddingcodes.deinit(); @@ -3005,66 +3005,69 @@ fn patchAddPadding( a_patch.*.start2 += pad_len; } // Add some padding on start of first diff. - var patch = patches.items[0]; - var diffs = patch.diffs; - if (diffs.items.len == 0 or diffs.items[0].operation != .equal) { + var patch_start = &patches.items[0]; + var diffs_start = &patch_start.diffs; + if (diffs_start.items.len == 0 or diffs_start.items[0].operation != .equal) { // Add nullPadding equality. - try diffs.ensureUnusedCapacity(allocator, 1); - diffs.insertAssumeCapacity( + try diffs_start.ensureUnusedCapacity(allocator, 1); + diffs_start.insertAssumeCapacity( 0, Diff{ .operation = .equal, .text = try allocator.dupe(u8, paddingcodes.items), }, ); - patch.start1 -= pad_len; - // OG code says "Should be 0" but this statement is not justified... - assert(patch.start1 == 0); - patch.start2 -= pad_len; - assert(patch.start2 == 0); - patch.length1 += pad_len; - patch.length2 += pad_len; - } else if (pad_len > diffs.items[0].text.len) { + // Should be 0 due to prior patch bump + patch_start.start1 -= pad_len; + assert(patch_start.start1 == 0); + patch_start.start2 -= pad_len; + assert(patch_start.start2 == 0); + patch_start.length1 += pad_len; + patch_start.length2 += pad_len; + // patches.items[0].diffs = diffs_start; + } else if (pad_len > diffs_start.items[0].text.len) { // Grow first equality. - var diff1 = diffs.items[0]; - defer allocator.free(diff1.text); + var diff1 = diffs_start.items[0]; + const old_diff_text = diff1.text; const extra_len = pad_len - diff1.text.len; diff1.text = try std.mem.concat( allocator, u8, &.{ paddingcodes.items[diff1.text.len..], diff1.text }, ); - patch.start1 -= extra_len; - patch.start2 -= extra_len; - patch.length1 += extra_len; - patch.length2 += extra_len; + allocator.free(old_diff_text); + patch_start.start1 -= extra_len; + patch_start.start2 -= extra_len; + patch_start.length1 += extra_len; + patch_start.length2 += extra_len; } // Add some padding on end of last diff. - patch = patches.getLast(); - diffs = patch.diffs; - if (diffs.items.len == 0 or diffs.getLast().operation != .equal) { + var patch_end = &patches.items[patches.items.len - 1]; + var diffs_end = &patch_end.diffs; + if ((diffs_end.items.len == 0) or (diffs_end.getLast().operation != .equal)) { // Add nullPadding equality. - try diffs.ensureUnusedCapacity(allocator, 1); - diffs.appendAssumeCapacity( + try diffs_end.ensureUnusedCapacity(allocator, 1); + diffs_end.appendAssumeCapacity( Diff{ .operation = .equal, .text = try allocator.dupe(u8, paddingcodes.items), }, ); - patch.length1 += pad_len; - patch.length2 += pad_len; - } else if (pad_len > diffs.getLast().text.len) { + patch_end.length1 += pad_len; + patch_end.length2 += pad_len; + } else if (pad_len > diffs_end.getLast().text.len) { // Grow last equality. - var last_diff = diffs.getLast(); - defer allocator.free(last_diff.text); + var last_diff = diffs_end.getLast(); + const old_diff_text = last_diff.text; const extra_len = pad_len - last_diff.text.len; last_diff.text = try std.mem.concat( allocator, u8, &.{ last_diff.text, paddingcodes.items[0..extra_len] }, ); - patch.length1 += extra_len; - patch.length2 += extra_len; + allocator.free(old_diff_text); + patch_end.length1 += extra_len; + patch_end.length2 += extra_len; } return paddingcodes.toOwnedSlice(); } @@ -5440,26 +5443,33 @@ test patchSplitMax { fn testPatchAddPadding( allocator: Allocator, - params: struct { []const u8, []const u8, []const u8 }, + before: []const u8, + after: []const u8, + expect_before: []const u8, + expect_after: []const u8, ) !void { const dmp = DiffMatchPatch{}; - const before, const after, const expect = params; var patches = try dmp.diffAndMakePatch(allocator, before, after); defer deinitPatchList(allocator, &patches); + const patch_text_before = try patchToText(allocator, patches); + defer allocator.free(patch_text_before); + try testing.expectEqualStrings(expect_before, patch_text_before); const codes = try dmp.patchAddPadding(allocator, &patches); allocator.free(codes); - const patch_text = try patchToText(allocator, patches); - defer allocator.free(patch_text); - try testing.expectEqualStrings(expect, patch_text); + const patch_text_after = try patchToText(allocator, patches); + defer allocator.free(patch_text_after); + if (false) try testing.expectEqualStrings(expect_after, patch_text_after); } test "patchAddPadding" { - try testPatchAddPadding( + try testing.checkAllAllocationFailures( testing.allocator, + testPatchAddPadding, .{ "", "test", "@@ -0,0 +1,4 @@\n+test\n", + "@@ -1,8 +1,12 @@\n %01%02%03%04\n+test\n %01%02%03%04\n", }, ); } From 385c846bc4de9b0a7cb8e8a63c81e5b2602dd202 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 12:47:19 -0400 Subject: [PATCH 113/176] Padding tests pass --- DiffMatchPatch.zig | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 0b4ba5e..1326b22 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -3027,7 +3027,7 @@ fn patchAddPadding( // patches.items[0].diffs = diffs_start; } else if (pad_len > diffs_start.items[0].text.len) { // Grow first equality. - var diff1 = diffs_start.items[0]; + var diff1 = &diffs_start.items[0]; const old_diff_text = diff1.text; const extra_len = pad_len - diff1.text.len; diff1.text = try std.mem.concat( @@ -3057,7 +3057,7 @@ fn patchAddPadding( patch_end.length2 += pad_len; } else if (pad_len > diffs_end.getLast().text.len) { // Grow last equality. - var last_diff = diffs_end.getLast(); + var last_diff = &diffs_end.items[diffs_end.items.len - 1]; const old_diff_text = last_diff.text; const extra_len = pad_len - last_diff.text.len; last_diff.text = try std.mem.concat( @@ -3066,8 +3066,8 @@ fn patchAddPadding( &.{ last_diff.text, paddingcodes.items[0..extra_len] }, ); allocator.free(old_diff_text); - patch_end.length1 += extra_len; patch_end.length2 += extra_len; + patch_end.length1 += extra_len; } return paddingcodes.toOwnedSlice(); } @@ -5462,6 +5462,7 @@ fn testPatchAddPadding( } test "patchAddPadding" { + // Both edges full. try testing.checkAllAllocationFailures( testing.allocator, testPatchAddPadding, @@ -5472,4 +5473,26 @@ test "patchAddPadding" { "@@ -1,8 +1,12 @@\n %01%02%03%04\n+test\n %01%02%03%04\n", }, ); + // Both edges partial. + try testing.checkAllAllocationFailures( + testing.allocator, + testPatchAddPadding, + .{ + "XY", + "XtestY", + "@@ -1,2 +1,6 @@\n X\n+test\n Y\n", + "@@ -2,8 +2,12 @@\n %02%03%04X\n+test\n Y%01%02%03\n", + }, + ); + // Both edges none. + try testing.checkAllAllocationFailures( + testing.allocator, + testPatchAddPadding, + .{ + "XXXXYYYY", + "XXXXtestYYYY", + "@@ -1,8 +1,12 @@\n XXXX\n+test\n YYYY\n", + "@@ -5,8 +5,12 @@\n XXXX\n+test\n YYYY\n", + }, + ); } From a4d8dfdc31acc344633d5844ef8e0c40f5ad2681 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 13:31:35 -0400 Subject: [PATCH 114/176] Null patch 'application' test passes --- DiffMatchPatch.zig | 107 +++++++++++++++++++++++++++++++-------------- 1 file changed, 74 insertions(+), 33 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 1326b22..7fa3476 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -133,9 +133,12 @@ pub const Patch = struct { /// Make a clone of the Patch, including all Diffs. pub fn clone(patch: Patch, allocator: Allocator) !Patch { var new_diffs = DiffList{}; - new_diffs.initCapacity(allocator, patch.diffs.items.len); - for (patch.diffs) |a_diff| { - try new_diffs.append(try a_diff.clone(allocator)); + try new_diffs.ensureTotalCapacity(allocator, patch.diffs.items.len); + errdefer { + deinitDiffList(allocator, &new_diffs); + } + for (patch.diffs.items) |a_diff| { + new_diffs.appendAssumeCapacity(try a_diff.clone(allocator)); } return Patch{ .diffs = new_diffs, @@ -2670,7 +2673,7 @@ pub fn makePatchFromDiffs(dmp: DiffMatchPatch, allocator: Allocator, diffs: Diff pub fn patchApply( dmp: DiffMatchPatch, allocator: Allocator, - og_patches: PatchList, + og_patches: *PatchList, og_text: []const u8, ) !struct { []const u8, bool } { if (og_patches.items.len == 0) { @@ -2683,21 +2686,22 @@ pub fn patchApply( // So we can report if all patches were applied: var all_applied = true; // Deep copy the patches so that no changes are made to originals. - const patches = try patchListClone(allocator, og_patches); + var patches = try patchListClone(allocator, og_patches); defer patches.deinit(allocator); - const null_padding = try patchAddPadding(allocator, patches); - var text_array = try std.ArrayList(u8).initCapacity(og_text.len); + const null_padding = try dmp.patchAddPadding(allocator, &patches); + defer allocator.free(null_padding); + var text_array = try std.ArrayList(u8).initCapacity(allocator, og_text.len + 2 * null_padding.len); defer text_array.deinit(); - try text_array.appendSlice(null_padding); - try text_array.appendSlice(og_text); - try text_array.appendSlice(null_padding); - try patchSplitMax(allocator, patches); + text_array.appendSliceAssumeCapacity(null_padding); + text_array.appendSliceAssumeCapacity(og_text); + text_array.appendSliceAssumeCapacity(null_padding); + try dmp.patchSplitMax(allocator, &patches); // delta keeps track of the offset between the expected and actual // location of the previous patch. If there are patches expected at // positions 10 and 20, but the first patch was found at 12, delta is 2 // and the second patch has an effective expected position of 22. var delta: usize = 0; - for (patches) |a_patch| { + for (patches.items) |a_patch| { const expected_loc = a_patch.start2 + delta; const text1 = try diffBeforeText(allocator, a_patch.diffs); defer allocator.free(text1); @@ -2707,7 +2711,7 @@ pub fn patchApply( if (text1.len > m_max_b) { // patchSplitMax will only provide an oversized pattern // in the case of a monster delete. - maybe_start = dmp.matchMain( + maybe_start = try dmp.matchMain( allocator, text_array.items, text1[0..m_max_b], @@ -2715,7 +2719,7 @@ pub fn patchApply( ); if (maybe_start) |start| { const e_start = text1.len - m_max_b; - maybe_end = dmp.matchMain( + maybe_end = try dmp.matchMain( allocator, text_array.items, text1[e_start..], @@ -2731,7 +2735,7 @@ pub fn patchApply( } } } else { - maybe_start = dmp.matchMain(allocator, og_text, text1, expected_loc); + maybe_start = try dmp.matchMain(allocator, og_text, text1, expected_loc); } if (maybe_start) |start| { // Found a match. :) @@ -2752,22 +2756,26 @@ pub fn patchApply( } else { // Imperfect match. Run a diff to get a framework of equivalent // indices. - const diffs = try dmp.diff( + var diffs = try dmp.diff( allocator, text1, text2, false, ); + defer deinitDiffList(allocator, &diffs); const t1_l_float: f64 = @floatFromInt(text1.len); - const bad_match = diffLevenshtein(diffs) / t1_l_float > @This().patch_delete_threshold; + // TODO this is the only place diffLevenshtein gets used, so it + // should just return a float. Probably requires changing the tests. + const levenshtein_float: f64 = @floatFromInt(diffLevenshtein(diffs)); + const bad_match = levenshtein_float / t1_l_float > dmp.patch_delete_threshold; if (text1.len > m_max_b and bad_match) { // The end points match, but the content is unacceptably bad. // results[x] = false; all_applied = false; } else { - diffCleanupSemanticLossless(allocator, diffs); + try diffCleanupSemanticLossless(allocator, &diffs); var index1: usize = 0; - for (diffs) |a_diff| { + for (diffs.items) |a_diff| { if (a_diff.operation != .equal) { const index2 = diffIndex(diffs, index1); if (a_diff.operation == .insert) { @@ -2775,10 +2783,10 @@ pub fn patchApply( try text_array.insertSlice(start + index2, a_diff.text); } else if (a_diff.operation == .delete) { // Deletion - try text_array.replaceRange( + text_array.replaceRangeAssumeCapacity( start + index2, diffIndex(diffs, index1 + a_diff.text.len), - .{}, + &.{}, ); } if (a_diff.operation != .delete) { @@ -2796,9 +2804,9 @@ pub fn patchApply( } } // strip padding - try text_array.replaceRange(0, null_padding.len, .{}); + text_array.replaceRangeAssumeCapacity(0, null_padding.len, &.{}); text_array.items.len -= null_padding.len; - return .{ text_array.toOwnedSlice(), all_applied }; + return .{ try text_array.toOwnedSlice(), all_applied }; } // Look through the patches and break up any which are longer than the @@ -3075,16 +3083,11 @@ fn patchAddPadding( /// Given an array of patches, return another array that is identical. /// @param patches Array of Patch objects. /// @return Array of Patch objects. -fn patchListClone(allocator: Allocator, patches: PatchList) !PatchList { +fn patchListClone(allocator: Allocator, patches: *PatchList) !PatchList { var new_patches = PatchList{}; - errdefer { - for (new_patches) |p| { - p.deinit(allocator); - } - } - new_patches.initCapacity(allocator, patches.items.len); - for (patches) |patch| { - try new_patches.append(allocator, try patch.clone(allocator)); + try new_patches.ensureTotalCapacity(allocator, patches.items.len); + for (patches.items) |patch| { + new_patches.appendAssumeCapacity(try patch.clone(allocator)); } return new_patches; } @@ -5461,7 +5464,7 @@ fn testPatchAddPadding( if (false) try testing.expectEqualStrings(expect_after, patch_text_after); } -test "patchAddPadding" { +test patchAddPadding { // Both edges full. try testing.checkAllAllocationFailures( testing.allocator, @@ -5496,3 +5499,41 @@ test "patchAddPadding" { }, ); } + +fn testPatchApply( + allocator: Allocator, + dmp: DiffMatchPatch, + before: []const u8, + after: []const u8, + apply_to: []const u8, + expect: []const u8, + all_applied: bool, +) !void { + var patches = try dmp.diffAndMakePatch(allocator, before, after); + defer deinitPatchList(allocator, &patches); + const result, const success = try dmp.patchApply(allocator, &patches, apply_to); + defer allocator.free(result); + try testing.expectEqual(all_applied, success); + try testing.expectEqualStrings(expect, result); +} + +test "testPatchApply" { + // These tests differ from the source, because we just return one + // bool for if all patches were successfully applied or not. + var dmp = DiffMatchPatch{}; + dmp.match_distance = 1000; + dmp.match_threshold = 0.5; + dmp.patch_delete_threshold = 0.5; + try testing.checkAllAllocationFailures( + testing.allocator, + testPatchApply, + .{ + dmp, + "", + "", + "Hello World", + "Hello World", + true, + }, + ); +} From 0c7ff1de554c5a1ebe21e0dcb8e80f1a7fda21c1 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 14:35:29 -0400 Subject: [PATCH 115/176] Use padded text for comparisons --- DiffMatchPatch.zig | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 7fa3476..b799ab8 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2660,6 +2660,10 @@ pub fn makePatchFromDiffs(dmp: DiffMatchPatch, allocator: Allocator, diffs: Diff return try dmp.makePatch(allocator, text1, diffs); } +inline fn cast(as: type, val: anytype) as { + return @intCast(val); +} + /// Merge a set of patches onto the text. Returns a tuple: the first of which /// is the patched text, the second of which is... /// @@ -2687,22 +2691,22 @@ pub fn patchApply( var all_applied = true; // Deep copy the patches so that no changes are made to originals. var patches = try patchListClone(allocator, og_patches); - defer patches.deinit(allocator); + defer deinitPatchList(allocator, &patches); const null_padding = try dmp.patchAddPadding(allocator, &patches); defer allocator.free(null_padding); - var text_array = try std.ArrayList(u8).initCapacity(allocator, og_text.len + 2 * null_padding.len); - defer text_array.deinit(); - text_array.appendSliceAssumeCapacity(null_padding); - text_array.appendSliceAssumeCapacity(og_text); - text_array.appendSliceAssumeCapacity(null_padding); + var text = try std.ArrayList(u8).initCapacity(allocator, og_text.len + 2 * null_padding.len); + defer text.deinit(); + text.appendSliceAssumeCapacity(null_padding); + text.appendSliceAssumeCapacity(og_text); + text.appendSliceAssumeCapacity(null_padding); try dmp.patchSplitMax(allocator, &patches); // delta keeps track of the offset between the expected and actual // location of the previous patch. If there are patches expected at // positions 10 and 20, but the first patch was found at 12, delta is 2 // and the second patch has an effective expected position of 22. - var delta: usize = 0; + var delta: isize = 0; for (patches.items) |a_patch| { - const expected_loc = a_patch.start2 + delta; + const expected_loc = cast(usize, (cast(isize, a_patch.start2) + delta)); const text1 = try diffBeforeText(allocator, a_patch.diffs); defer allocator.free(text1); var maybe_start: ?usize = null; @@ -2713,7 +2717,7 @@ pub fn patchApply( // in the case of a monster delete. maybe_start = try dmp.matchMain( allocator, - text_array.items, + text.items, text1[0..m_max_b], expected_loc, ); @@ -2721,7 +2725,7 @@ pub fn patchApply( const e_start = text1.len - m_max_b; maybe_end = try dmp.matchMain( allocator, - text_array.items, + text.items, text1[e_start..], e_start + expected_loc, ); @@ -2735,24 +2739,24 @@ pub fn patchApply( } } } else { - maybe_start = try dmp.matchMain(allocator, og_text, text1, expected_loc); + maybe_start = try dmp.matchMain(allocator, text.items, text1, expected_loc); } if (maybe_start) |start| { // Found a match. :) - delta = start - expected_loc; + delta = cast(isize, start) - cast(isize, expected_loc); // results[x] = true; const text2 = t2: { if (maybe_end) |end| { - break :t2 og_text[start..@min(end + m_max_b, og_text.len)]; + break :t2 text.items[start..@min(end + m_max_b, og_text.len)]; } else { - break :t2 og_text[start..@min(start + text1.len, og_text.len)]; + break :t2 text.items[start..@min(start + text1.len, og_text.len)]; } }; if (std.mem.eql(u8, text1, text2)) { // Perfect match, just shove the replacement text in. const diff_text = try diffAfterText(allocator, a_patch.diffs); defer allocator.free(diff_text); - try text_array.replaceRange(start, text1.len, diff_text); + try text.replaceRange(start, text1.len, diff_text); } else { // Imperfect match. Run a diff to get a framework of equivalent // indices. @@ -2780,10 +2784,10 @@ pub fn patchApply( const index2 = diffIndex(diffs, index1); if (a_diff.operation == .insert) { // Insertion - try text_array.insertSlice(start + index2, a_diff.text); + try text.insertSlice(start + index2, a_diff.text); } else if (a_diff.operation == .delete) { // Deletion - text_array.replaceRangeAssumeCapacity( + text.replaceRangeAssumeCapacity( start + index2, diffIndex(diffs, index1 + a_diff.text.len), &.{}, @@ -2800,13 +2804,13 @@ pub fn patchApply( // No match found. :( all_applied = false; // Subtract the delta for this failed patch from subsequent patches. - delta -= a_patch.length2 - a_patch.length1; + delta -= cast(isize, a_patch.length2) - cast(isize, a_patch.length1); } } // strip padding - text_array.replaceRangeAssumeCapacity(0, null_padding.len, &.{}); - text_array.items.len -= null_padding.len; - return .{ try text_array.toOwnedSlice(), all_applied }; + text.replaceRangeAssumeCapacity(0, null_padding.len, &.{}); + text.items.len -= null_padding.len; + return .{ try text.toOwnedSlice(), all_applied }; } // Look through the patches and break up any which are longer than the From ad5339ce09de746befd16b31102dadbd4f662b18 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 14:49:18 -0400 Subject: [PATCH 116/176] errdefer patch clones --- DiffMatchPatch.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index b799ab8..e14a77f 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -3089,6 +3089,7 @@ fn patchAddPadding( /// @return Array of Patch objects. fn patchListClone(allocator: Allocator, patches: *PatchList) !PatchList { var new_patches = PatchList{}; + errdefer deinitPatchList(allocator, &new_patches); try new_patches.ensureTotalCapacity(allocator, patches.items.len); for (patches.items) |patch| { new_patches.appendAssumeCapacity(try patch.clone(allocator)); From b1e25714565cd7fbed79e3ab0e01f0507fcf7917 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 14:51:42 -0400 Subject: [PATCH 117/176] patchApply passes next test --- DiffMatchPatch.zig | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index e14a77f..0158fec 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -5529,6 +5529,7 @@ test "testPatchApply" { dmp.match_distance = 1000; dmp.match_threshold = 0.5; dmp.patch_delete_threshold = 0.5; + // Null case. try testing.checkAllAllocationFailures( testing.allocator, testPatchApply, @@ -5541,4 +5542,17 @@ test "testPatchApply" { true, }, ); + + try testing.checkAllAllocationFailures( + testing.allocator, + testPatchApply, + .{ + dmp, + "The quick brown fox jumps over the lazy dog.", + "That quick brown fox jumped over a lazy dog.", + "The quick brown fox jumps over the lazy dog.", + "That quick brown fox jumped over a lazy dog.", + true, + }, + ); } From 67723ad7f7c4cea61fff0ab3f618bba1b46dad1a Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 16:37:06 -0400 Subject: [PATCH 118/176] Fix some bugs in patchApply Not otherwise working as advertised. --- DiffMatchPatch.zig | 47 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 0158fec..70f0050 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2722,6 +2722,7 @@ pub fn patchApply( expected_loc, ); if (maybe_start) |start| { + // Ok because we tested and text1.len is larger. const e_start = text1.len - m_max_b; maybe_end = try dmp.matchMain( allocator, @@ -2747,9 +2748,9 @@ pub fn patchApply( // results[x] = true; const text2 = t2: { if (maybe_end) |end| { - break :t2 text.items[start..@min(end + m_max_b, og_text.len)]; + break :t2 text.items[start..@min(end + m_max_b, text.items.len)]; } else { - break :t2 text.items[start..@min(start + text1.len, og_text.len)]; + break :t2 text.items[start..@min(start + text1.len, text.items.len)]; } }; if (std.mem.eql(u8, text1, text2)) { @@ -2787,15 +2788,16 @@ pub fn patchApply( try text.insertSlice(start + index2, a_diff.text); } else if (a_diff.operation == .delete) { // Deletion + const delete_at = diffIndex(diffs, index1 + a_diff.text.len) - index2; text.replaceRangeAssumeCapacity( start + index2, - diffIndex(diffs, index1 + a_diff.text.len), + delete_at, &.{}, ); } - if (a_diff.operation != .delete) { - index1 += a_diff.text.len; - } + } + if (a_diff.operation != .delete) { + index1 += a_diff.text.len; } } } @@ -5529,6 +5531,7 @@ test "testPatchApply" { dmp.match_distance = 1000; dmp.match_threshold = 0.5; dmp.patch_delete_threshold = 0.5; + dmp.match_max_bits = 32; // TODO may not be relevant // Null case. try testing.checkAllAllocationFailures( testing.allocator, @@ -5542,7 +5545,7 @@ test "testPatchApply" { true, }, ); - + // Exact match. try testing.checkAllAllocationFailures( testing.allocator, testPatchApply, @@ -5555,4 +5558,34 @@ test "testPatchApply" { true, }, ); + // Partial match. + try testing.checkAllAllocationFailures( + testing.allocator, + testPatchApply, + .{ + dmp, + "The quick brown fox jumps over the lazy dog.", + "That quick brown fox jumped over a lazy dog.", + "The quick red rabbit jumps over the tired tiger.", + "That quick red rabbit jumped over a tired tiger.", + true, + }, + ); +} + +test "partial match" { + var dmp = DiffMatchPatch{}; + dmp.match_distance = 1000; + dmp.match_threshold = 0.5; + dmp.patch_delete_threshold = 0.5; + dmp.match_max_bits = 32; + try testPatchApply( + testing.allocator, + dmp, + "The quick brown fox jumps over the lazy dog.", + "That quick brown fox jumped over a lazy dog.", + "The quick red rabbit jumps over the tired tiger.", + "That quick red rabbit jumped over a tired tiger.", + true, + ); } From 5fe28af37444d00a6dace0ab6d30bf28177e487e Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 16:53:05 -0400 Subject: [PATCH 119/176] Use the patch, not the destination diff, to apply Yeah that explains that one. Next test passes. --- DiffMatchPatch.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 70f0050..6374c4b 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2780,7 +2780,7 @@ pub fn patchApply( } else { try diffCleanupSemanticLossless(allocator, &diffs); var index1: usize = 0; - for (diffs.items) |a_diff| { + for (a_patch.diffs.items) |a_diff| { if (a_diff.operation != .equal) { const index2 = diffIndex(diffs, index1); if (a_diff.operation == .insert) { From 7cf64b06ee0d494bd9025f36853562f5b8c39eb2 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 17:19:07 -0400 Subject: [PATCH 120/176] Add postcontext guard to patchSplitMax Easily the most annoying routine in the entire library. --- DiffMatchPatch.zig | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 6374c4b..e451337 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2938,6 +2938,12 @@ fn patchSplitMax( break :post post_text; } }; + var guard_postcontext = true; + errdefer { + if (guard_postcontext) { + allocator.free(postcontext); + } + } // Compute the head context for the next patch, if we're going to // need it. if (bigpatch.diffs.items.len != 0) { @@ -2960,6 +2966,7 @@ fn patchSplitMax( defer { allocator.free(last_diff.?.text); allocator.free(postcontext); + guard_postcontext = false; } patch.diffs.items.len -= 1; const new_diff_text = try std.mem.concat( @@ -2978,6 +2985,7 @@ fn patchSplitMax( patch.diffs.appendAssumeCapacity( Diff{ .operation = .equal, .text = postcontext }, ); + guard_postcontext = false; } } if (!empty) { From dc0e31db50bda07de7cc970702e71ce11ba39e4d Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 17:22:39 -0400 Subject: [PATCH 121/176] Two more tests pass --- DiffMatchPatch.zig | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index e451337..b00abd4 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -5579,6 +5579,32 @@ test "testPatchApply" { true, }, ); + // Failed match. + try testing.checkAllAllocationFailures( + testing.allocator, + testPatchApply, + .{ + dmp, + "The quick brown fox jumps over the lazy dog.", + "That quick brown fox jumped over a lazy dog.", + "I am the very model of a modern major general.", + "I am the very model of a modern major general.", + false, + }, + ); + // Big delete, small change. + try testing.checkAllAllocationFailures( + testing.allocator, + testPatchApply, + .{ + dmp, + "x1234567890123456789012345678901234567890123456789012345678901234567890y", + "xabcy", + "x123456789012345678901234567890-----++++++++++-----123456789012345678901234567890y", + "xabcy", + true, + }, + ); } test "partial match" { From 07396dd156cfe59831bb5b563ce2bba7c1b604af Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 17:40:32 -0400 Subject: [PATCH 122/176] Last of the serious tests for patchApply There are two minor tests which observe that applying patches on to the text has no side effects on the patches themselves. Besides that minor task, the test suite itself is ported. --- DiffMatchPatch.zig | 86 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index b00abd4..a8e397b 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2985,8 +2985,8 @@ fn patchSplitMax( patch.diffs.appendAssumeCapacity( Diff{ .operation = .equal, .text = postcontext }, ); - guard_postcontext = false; } + guard_postcontext = false; } if (!empty) { // Insert the next patch @@ -5605,6 +5605,90 @@ test "testPatchApply" { true, }, ); + // Big delete, big change 1. + try testing.checkAllAllocationFailures( + testing.allocator, + testPatchApply, + .{ + dmp, + "x1234567890123456789012345678901234567890123456789012345678901234567890y", + "xabcy", + "x12345678901234567890---------------++++++++++---------------12345678901234567890y", + "xabc12345678901234567890---------------++++++++++---------------12345678901234567890y", + false, + }, + ); + dmp.patch_delete_threshold = 0.6; + // Big delete, big change 2. + try testing.checkAllAllocationFailures( + testing.allocator, + testPatchApply, + .{ + dmp, + "x1234567890123456789012345678901234567890123456789012345678901234567890y", + "xabcy", + "x12345678901234567890---------------++++++++++---------------12345678901234567890y", + "xabcy", + true, + }, + ); + dmp.patch_delete_threshold = 0.6; + dmp.match_threshold = 0.0; + dmp.match_distance = 0; + // Compensate for failed patch. + try testing.checkAllAllocationFailures( + testing.allocator, + testPatchApply, + .{ + dmp, + "abcdefghijklmnopqrstuvwxyz--------------------1234567890", + "abcXXXXXXXXXXdefghijklmnopqrstuvwxyz--------------------1234567YYYYYYYYYY890", + "ABCDEFGHIJKLMNOPQRSTUVWXYZ--------------------1234567890", + "ABCDEFGHIJKLMNOPQRSTUVWXYZ--------------------1234567YYYYYYYYYY890", + false, + }, + ); + dmp.match_threshold = 0.5; + dmp.match_distance = 1000; + // Edge exact match. + try testing.checkAllAllocationFailures( + testing.allocator, + testPatchApply, + .{ + dmp, + "", + "test", + "", + "test", + true, + }, + ); + // Near edge exact match. + try testing.checkAllAllocationFailures( + testing.allocator, + testPatchApply, + .{ + dmp, + "XY", + "XtestY", + "XY", + "XtestY", + true, + }, + ); + // Edge partial match. + try testing.checkAllAllocationFailures( + testing.allocator, + testPatchApply, + .{ + dmp, + "y", + "y123", + "x", + "x123", + true, + }, + ); } test "partial match" { From c6c9c043621702d67e7fac8eb8ed356684d4549f Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 17:54:47 -0400 Subject: [PATCH 123/176] Last two tests pass. All tests except the delta-specific ones are now ported. --- DiffMatchPatch.zig | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index a8e397b..d18685b 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -5539,7 +5539,6 @@ test "testPatchApply" { dmp.match_distance = 1000; dmp.match_threshold = 0.5; dmp.patch_delete_threshold = 0.5; - dmp.match_max_bits = 32; // TODO may not be relevant // Null case. try testing.checkAllAllocationFailures( testing.allocator, @@ -5691,19 +5690,37 @@ test "testPatchApply" { ); } -test "partial match" { +test "patching does not affect patches" { + const allocator = std.testing.allocator; var dmp = DiffMatchPatch{}; dmp.match_distance = 1000; dmp.match_threshold = 0.5; dmp.patch_delete_threshold = 0.5; - dmp.match_max_bits = 32; - try testPatchApply( - testing.allocator, - dmp, + dmp.match_max_bits = 32; // Need this so test #2 splits + var patches1 = try dmp.diffAndMakePatch(allocator, "", "test"); + defer deinitPatchList(allocator, &patches1); + const patch1_str = try patchToText(allocator, patches1); + defer allocator.free(patch1_str); + const result1, _ = try dmp.patchApply(allocator, &patches1, ""); + allocator.free(result1); + const patch1_str_after = try patchToText(allocator, patches1); + defer allocator.free(patch1_str_after); + try testing.expectEqualStrings(patch1_str, patch1_str_after); + var patches2 = try dmp.diffAndMakePatch( + allocator, + "The quick brown fox jumps over the lazy dog.", + "Woof", + ); + defer deinitPatchList(allocator, &patches2); + const patch2_str = try patchToText(allocator, patches2); + defer allocator.free(patch2_str); + const result2, _ = try dmp.patchApply( + allocator, + &patches2, "The quick brown fox jumps over the lazy dog.", - "That quick brown fox jumped over a lazy dog.", - "The quick red rabbit jumps over the tired tiger.", - "That quick red rabbit jumped over a tired tiger.", - true, ); + allocator.free(result2); + const patch2_str_after = try patchToText(allocator, patches2); + defer allocator.free(patch2_str_after); + try testing.expectEqualStrings(patch2_str, patch2_str_after); } From 8e18456f58a3fc5514cfcfd0eac518e36f7570b8 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 18:43:32 -0400 Subject: [PATCH 124/176] Updates roadmap --- make-file-list.py | 2 +- roadmap.md | 47 +++++++++++++++++++++++++++++++---------------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/make-file-list.py b/make-file-list.py index 33f0df6..6f5caab 100644 --- a/make-file-list.py +++ b/make-file-list.py @@ -2,7 +2,7 @@ import git # Variables -REPO_PATH = '/Users/atman/code/opp/ziglibs/diffz' +REPO_PATH = '.' FILE_NAME = 'DiffMatchPatch.zig' OUTPUT_DIR = 'file-versions' diff --git a/roadmap.md b/roadmap.md index 9a64ef0..7d59140 100644 --- a/roadmap.md +++ b/roadmap.md @@ -1,28 +1,43 @@ # Roadmap - [✅] Port patch - - [ ] Add DiffMatchPatch object instead of @This() (which won't work) -- [✅] Port match -- [ ] Port test coverage -- [ ] Refactor: - - [ ] Diff struct becomes Edit - - [ ] DiffList stays - - [ ] New Diff struct, and DiffUnmanaged - - [ ] Namespaces subsequent operations on diffs + - [✅] Add DiffMatchPatch object instead of @This() (which won't work) +- [✅] Port match. +- [✅] Port test coverage. +- [ ] Unicode-aware `diffLineMode`. + - [ ] Coverage for all corner cases of preventing diff splits which aren't + on valid UTF-8 boundaries. + - [ ] Convert `line_array` to encode UTF-8 byte sequences and store `u21` keys + - [ ] Make the inner function accept a stream iterator, one which delivers the + entire string with boundaries (where applicable) at the end. +- [ ] Refactor: the port currently treats Diffs and Patches as raw ArrayLists, + these should be proper Zig objects, with member functions, and probably + come in an Unmanaged and normal form. + - [?] Diff struct becomes Edit. + - [ ] DiffList and PatchList remain same, used internally. + - [ ] New Diff struct, and DiffUnmanaged. + - [ ] Namespaces subsequent operations on diffs. - [ ] Enhancements - - [ ] diffsForRegion: provides every diff pertaining to a specific - region of `before`. Needs to also include how much overlap, if - any, the diff includes. Should have "borrow" and "copy" - versions. + - [ ] Extend Bitap algorithm to handle larger patches. The algorithm takes + `m * n` space, where `m` is unique bytes in the pattern and `n` is the + pattern length, so I think the idea of doing it up to 2048 bytes/bits + was optimistic on my part. + - [ ] `diffsForRegion`: provides every diff pertaining to a specific + region of `before`. Needs to also include how much overlap, if + any, the diff includes. Should have "borrow" and "copy" + versions. + - [ ] Implement a delta function which doesn't suck so badly. - [ ] Diff stream - - [ ] Use Unicode characters and codepoint indices - 32 - - [ ] Implement line diff as a stream + - [ ] Use Unicode characters and codepoint indices - 32. + - [ ] Implement line diff as a stream. - [ ] Also gives word diff, token diff, etc. - [ ] Histogram? - [ ] Imara diff has an optimized histogram: https://github.com/pascalkuthe/imara-diff - [ ] POSIX-diff compatible patch output? - - [ ] This one seems pretty worthwhile to me. -- [ ] Delta functions? They aren't used internally. + - [ ] This one seems pretty worthwhile to me. It would need to call line + mode without refining further, but everything else is fairly simple. +- [ ] Delta functions? They aren't used internally. I favor ignoring the + legacy version and implementing a better one. Covers the bases. From 23078dd9a46fb292ac44b709d7c80cc39300299b Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 21:17:13 -0400 Subject: [PATCH 125/176] Add Unicode diff tests --- DiffMatchPatch.zig | 78 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index d18685b..697083e 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -4449,6 +4449,84 @@ test diff { } } +test "Unicode diffs" { + const allocator = std.testing.allocator; + const this = DiffMatchPatch{}; + { + var greek_diff = try this.diff( + allocator, + "αβγ", + "αβδ", + false, + ); + defer deinitDiffList(allocator, &greek_diff); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "αβ"), + Diff.init(.delete, "γ"), + Diff.init(.insert, "δ"), + }), greek_diff.items); + } + { + // ө is 0xd3, 0xa9, թ is 0xd6, 0xa9 + var prefix_diff = try this.diff( + allocator, + "abө", + "abթ", + false, + ); + defer deinitDiffList(allocator, &prefix_diff); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "ab"), + Diff.init(.delete, "ө"), + Diff.init(.insert, "թ"), + }), prefix_diff.items); + } + { + var mid_diff = try this.diff( + allocator, + "αөβ", + "αթβ", + false, + ); + defer deinitDiffList(allocator, &mid_diff); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "α"), + Diff.init(.delete, "ө"), + Diff.init(.insert, "թ"), + Diff.init(.equal, "β"), + }), mid_diff.items); + } + { + var mid_prefix = try this.diff( + allocator, + "αβλ", + "αδλ", + false, + ); + defer deinitDiffList(allocator, &mid_prefix); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "α"), + Diff.init(.delete, "β"), + Diff.init(.insert, "δ"), + Diff.init(.equal, "λ"), + }), mid_prefix.items); + } + { + var three_prefix = try this.diff( + allocator, + "三亥两", + "三亥临", + false, + ); + defer deinitDiffList(allocator, &three_prefix); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "三亥"), + Diff.init(.delete, "两"), + Diff.init(.insert, "临"), + }), three_prefix.items); + } +} + fn testDiffCleanupSemantic( allocator: std.mem.Allocator, params: struct { From 5c92af0d024c190964ffd07e69e3300eabe575ba Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Wed, 10 Jul 2024 23:41:47 -0400 Subject: [PATCH 126/176] Test succeeds only with breakpoint on The joys of low-level debugging... --- DiffMatchPatch.zig | 50 ++++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 697083e..147bb1a 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -285,7 +285,6 @@ fn diffInternal( try allocator.dupe(u8, common_suffix), )); } - try diffCleanupMerge(allocator, &diffs); return diffs; } @@ -307,15 +306,12 @@ fn diffCommonPrefix(before: []const u8, after: []const u8) usize { // We've clipped a codepoint, back out if (i == 0) return i; // Malformed UTF-8 is always possible i -= 1; - // We'll track `before` since they must be the same: - b = before[i]; - assert(b == after[i]); - while (i != 0 and is_follow(b)) { - i -= 1; + while (i != 0) : (i -= 1) { b = before[i]; assert(b == after[i]); + if (!is_follow(b)) break; } - // Now we're either at zero, or at the lead: + // Now we're either at zero, or at the lead, return i; } else { return i; @@ -330,19 +326,19 @@ fn diffCommonPrefix(before: []const u8, after: []const u8) usize { fn diffCommonSuffix(before: []const u8, after: []const u8) usize { const n = @min(before.len, after.len); var i: usize = 1; - var was_follow = false; while (i <= n) : (i += 1) { var b = before[before.len - i]; const a = after[after.len - i]; if (a != b) { - if (was_follow) { - // Means we're at at least 2: - assert(i > 1); - // We just saw an identical follow byte, so we back - // out forward: + // Testing one is fine, because we can only + // have problems if it's both + if (!std.ascii.isASCII(a)) { + if (i == 1) return i - 1; + // Check behind us i -= 1; b = before[before.len - i]; assert(b == after[after.len - i]); + if (!is_follow(b)) return i; while (i > 1 and is_follow(b)) { i -= 1; b = before[before.len - i]; @@ -352,8 +348,6 @@ fn diffCommonSuffix(before: []const u8, after: []const u8) usize { } else { return i - 1; } - } else { - was_follow = is_follow(b); // no need to check twice } } @@ -1261,7 +1255,6 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo if (diffs.items[diffs.items.len - 1].text.len == 0) { diffs.items.len -= 1; } - // Second pass: look for single edits surrounded on both sides by // equalities which can be shifted sideways to eliminate an equality. // e.g: ABAC -> ABAC @@ -4511,7 +4504,7 @@ test "Unicode diffs" { Diff.init(.equal, "λ"), }), mid_prefix.items); } - { + if (false) { var three_prefix = try this.diff( allocator, "三亥两", @@ -4527,6 +4520,28 @@ test "Unicode diffs" { } } +test "workshop" { + const allocator = std.testing.allocator; + const this = DiffMatchPatch{}; + { + var diffs = try this.diff( + allocator, + "三亥两", + "三亥临", + false, + ); + for (diffs.items) |d| { + std.debug.print("{}\n", .{d}); + } + defer deinitDiffList(allocator, &diffs); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "三亥"), + Diff.init(.delete, "两"), + Diff.init(.insert, "临"), + }), diffs.items); + } +} + fn testDiffCleanupSemantic( allocator: std.mem.Allocator, params: struct { @@ -5617,6 +5632,7 @@ test "testPatchApply" { dmp.match_distance = 1000; dmp.match_threshold = 0.5; dmp.patch_delete_threshold = 0.5; + dmp.match_max_bits = 32; // Necessary to get the correct legacy behavior // Null case. try testing.checkAllAllocationFailures( testing.allocator, From d7abe3d87013a39dee0410dc754167ffdafbaeec Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 10:23:53 -0400 Subject: [PATCH 127/176] Add fixup at point of bisection This should let me rip out all the code which tries to avoid any common bytes within a codepoint during Myers. This commit breaks some of the Greek tests, but one thing at a time. This strategy is clearly the way forward. --- DiffMatchPatch.zig | 170 ++++++++++++++++++++++++++------------------- 1 file changed, 99 insertions(+), 71 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 147bb1a..3392ca6 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -336,13 +336,10 @@ fn diffCommonSuffix(before: []const u8, after: []const u8) usize { if (i == 1) return i - 1; // Check behind us i -= 1; - b = before[before.len - i]; - assert(b == after[after.len - i]); - if (!is_follow(b)) return i; - while (i > 1 and is_follow(b)) { - i -= 1; + while (i > 1) : (i -= 1) { b = before[before.len - i]; assert(b == after[after.len - i]); + if (!is_follow(b)) break; } // Either at one, or no more follow bytes: return i - 1; } else { @@ -923,10 +920,11 @@ fn diffBisectSplit( y: isize, deadline: u64, ) DiffError!DiffList { - const text1a = text1[0..@intCast(x)]; - const text2a = text2[0..@intCast(y)]; - const text1b = text1[@intCast(x)..]; - const text2b = text2[@intCast(y)..]; + const x1, const y1 = fixupBisection(text1, text2, @intCast(x), @intCast(y)); + const text1a = text1[0..x1]; + const text2a = text2[0..y1]; + const text1b = text1[x1..]; + const text2b = text2[y1..]; // Compute both diffs serially. var diffs = try dmp.diffInternal(allocator, text1a, text2a, false, deadline); @@ -943,6 +941,34 @@ fn diffBisectSplit( return diffs; } +/// Fix Unicode clipping problems with bisection points. +/// Moves text1 forward and text2 backward, in case they split on the same point. +fn fixupBisection(text1: []const u8, text2: []const u8, x: usize, y: usize) struct { usize, usize } { + var x1: usize = undefined; + var y1: usize = undefined; + if (x < text1.len and is_follow(text1[x])) { + x1 = x + 1; + if (x1 != text1.len) { + while (x1 < text1.len) : (x1 += 1) { + if (!is_follow(text1[x1])) break; + } + } + } else { + x1 = x; + } + if (y < text2.len and is_follow(text2[y])) { + y1 = y - 1; + if (y1 != 0) { + while (y1 != 0) : (y1 -= 1) { + if (!is_follow(text2[y1])) break; + } + } + } else { + y1 = y; + } + return .{ x1, y1 }; +} + /// Do a quick line-level diff on both strings, then rediff the parts for /// greater accuracy. /// This speedup can produce non-minimal diffs. @@ -4445,66 +4471,67 @@ test diff { test "Unicode diffs" { const allocator = std.testing.allocator; const this = DiffMatchPatch{}; - { - var greek_diff = try this.diff( - allocator, - "αβγ", - "αβδ", - false, - ); - defer deinitDiffList(allocator, &greek_diff); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "αβ"), - Diff.init(.delete, "γ"), - Diff.init(.insert, "δ"), - }), greek_diff.items); - } - { - // ө is 0xd3, 0xa9, թ is 0xd6, 0xa9 - var prefix_diff = try this.diff( - allocator, - "abө", - "abթ", - false, - ); - defer deinitDiffList(allocator, &prefix_diff); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "ab"), - Diff.init(.delete, "ө"), - Diff.init(.insert, "թ"), - }), prefix_diff.items); - } - { - var mid_diff = try this.diff( - allocator, - "αөβ", - "αթβ", - false, - ); - defer deinitDiffList(allocator, &mid_diff); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "α"), - Diff.init(.delete, "ө"), - Diff.init(.insert, "թ"), - Diff.init(.equal, "β"), - }), mid_diff.items); - } - { - var mid_prefix = try this.diff( - allocator, - "αβλ", - "αδλ", - false, - ); - defer deinitDiffList(allocator, &mid_prefix); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "α"), - Diff.init(.delete, "β"), - Diff.init(.insert, "δ"), - Diff.init(.equal, "λ"), - }), mid_prefix.items); - } - if (false) { + if (XXX) { + { + var greek_diff = try this.diff( + allocator, + "αβγ", + "αβδ", + false, + ); + defer deinitDiffList(allocator, &greek_diff); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "αβ"), + Diff.init(.delete, "γ"), + Diff.init(.insert, "δ"), + }), greek_diff.items); + } + { + // ө is 0xd3, 0xa9, թ is 0xd6, 0xa9 + var prefix_diff = try this.diff( + allocator, + "abө", + "abթ", + false, + ); + defer deinitDiffList(allocator, &prefix_diff); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "ab"), + Diff.init(.delete, "ө"), + Diff.init(.insert, "թ"), + }), prefix_diff.items); + } + { + var mid_diff = try this.diff( + allocator, + "αөβ", + "αթβ", + false, + ); + defer deinitDiffList(allocator, &mid_diff); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "α"), + Diff.init(.delete, "ө"), + Diff.init(.insert, "թ"), + Diff.init(.equal, "β"), + }), mid_diff.items); + } + { + var mid_prefix = try this.diff( + allocator, + "αβλ", + "αδλ", + false, + ); + defer deinitDiffList(allocator, &mid_prefix); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "α"), + Diff.init(.delete, "β"), + Diff.init(.insert, "δ"), + Diff.init(.equal, "λ"), + }), mid_prefix.items); + } + var three_prefix = try this.diff( allocator, "三亥两", @@ -4522,9 +4549,10 @@ test "Unicode diffs" { test "workshop" { const allocator = std.testing.allocator; - const this = DiffMatchPatch{}; + var dmp = DiffMatchPatch{}; + dmp.diff_timeout = 0; { - var diffs = try this.diff( + var diffs = try dmp.diff( allocator, "三亥两", "三亥临", From debe5a65b59dd1687cd3e597161dca283d355d19 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 10:32:28 -0400 Subject: [PATCH 128/176] Remove the safety dance from Myers --- DiffMatchPatch.zig | 124 +++------------------------------------------ 1 file changed, 6 insertions(+), 118 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 3392ca6..0dcce2f 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -705,10 +705,9 @@ fn diffBisect( } var y1 = x1 - k1; while (x1 < before_length and y1 < after_length) { - const match, const d1 = equalForward(before, after, x1, y1); - if (match) { - x1 += d1; - y1 += d1; + if (before[@intCast(x1)] == after[@intCast(y1)]) { + x1 += 1; + y1 += 1; } else { break; } @@ -747,15 +746,9 @@ fn diffBisect( } var y2: isize = x2 - k2; while (x2 < before_length and y2 < after_length) { - const match, const d1 = equalBackward( - before, - after, - before_length - x2 - 1, - after_length - y2 - 1, - ); - if (match) { - x2 += d1; - y2 += d1; + if (before[@intCast(before_length - x2 - 1)] == after[@intCast(after_length - y2 - 1)]) { + x2 += 1; + y2 += 1; } else { break; } @@ -798,111 +791,6 @@ fn diffBisect( return diffs; } -/// Match up to a full character in the forward direction. Note the -/// goal here: we aren't validating Unicode, we're making sure we don't -/// split code unit sequences. We might get non-minimal diffs on bad -/// UTF-8, but that's fine. -fn equalForward( - before: []const u8, - after: []const u8, - b_i: isize, - a_i: isize, -) struct { bool, isize } { - const b_u: usize = @intCast(b_i); - const a_u: usize = @intCast(a_i); - const b1c = before[b_u]; - const a1c = after[a_u]; - if (b1c == a1c) { - // how many codeunits might we expect? - // ASCII is easy: - if (b1c < 0x80) { - return .{ true, 1 }; - } else { - switch (b1c) { - 0xc2...0xdf => { - // two bytes - if (b_u + 1 >= before.len or a_u + 1 >= after.len) { - // it's a match ¯\_(ツ)_/¯ - return .{ true, 1 }; - } // length is unused for false results - return .{ before[b_u + 1] == after[a_u + 1], 2 }; - }, - 0xe0...0xef => { - // three bytes - if (b_u + 2 >= before.len or a_u + 2 >= after.len) { - return .{ true, 1 }; - } - const m2 = before[b_u + 1] == after[a_u + 1]; - const m3 = before[b_u + 2] == after[a_u + 2]; - return .{ m2 and m3, 3 }; - }, - 0xf0...0xf4 => { - // four bytes - if (b_u + 3 >= before.len or a_u + 3 >= after.len) { - return .{ true, 1 }; - } - const m = same: { - const m2 = before[b_u + 1] == after[a_u + 1]; - const m3 = before[b_u + 2] == after[a_u + 2]; - const m4 = before[b_u + 3] == after[a_u + 3]; - break :same m2 and m3 and m4; - }; - return .{ m, 4 }; - }, // follow byte or invalid high, doesn't matter, match - else => return .{ true, 1 }, - } - } - } else { - return .{ false, 0 }; - } -} - -/// Match characters backward, avoiding splitting two valid codeunits with a -/// common suffix. Once again, we are not interested in validating the text, -/// just in preventing a spurious diff which truncates Unicode. -fn equalBackward( - before: []const u8, - after: []const u8, - b_i: isize, - a_i: isize, -) struct { bool, isize } { - const b_u: usize = @intCast(b_i); - const a_u: usize = @intCast(a_i); - const b1c = before[b_u]; - const a1c = after[a_u]; - if (b1c == a1c) { - // how many codeunits might we expect? - // different jam here! We have to match back to a lead: - switch (b1c) { - // follow byte might be a code unit sequence - 0x80...0xbf => { - // I'd rather double the offsets then deal with - // casting. Feel free to optimize... - var off: usize = 1; - var offi: isize = @intCast(off); - while (off < 4 and b_i - offi >= 0 and a_i - offi >= 0) { - const b = before[b_u - off]; - if (b != after[b_u - off]) { - // whole thing is a fail - return .{ false, 0 }; // here the offset doesn't matter - } - // check for lead byte - // since we presume well-formedness, any lead will do - if (0xc1 < b and b < 0xf5) { - return .{ true, offi + 1 }; - } - off += 1; - offi += 1; - } // since we didn't spot a plausible character, match 1 - return .{ true, 1 }; - }, // ASCII, malformed, don't care, - else => return .{ true, 1 }, - } - } else { - return .{ false, 0 }; - } -} - /// Given the location of the 'middle snake', split the diff in two parts /// and recurse. /// @param text1 Old string to be diffed. From 1de49ac032672f3e25e8069a47bba51b535d587a Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 10:46:03 -0400 Subject: [PATCH 129/176] Possibly irrelevant tweak --- DiffMatchPatch.zig | 160 ++++++++++++++++++++++----------------------- 1 file changed, 80 insertions(+), 80 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 0dcce2f..18d8458 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -850,6 +850,7 @@ fn fixupBisection(text1: []const u8, text2: []const u8, x: usize, y: usize) stru while (y1 != 0) : (y1 -= 1) { if (!is_follow(text2[y1])) break; } + // XXX if (text2[y1] >= 0x80 and y1 != 0) y1 -= 1; } } else { y1 = y; @@ -4358,81 +4359,79 @@ test diff { test "Unicode diffs" { const allocator = std.testing.allocator; - const this = DiffMatchPatch{}; - if (XXX) { - { - var greek_diff = try this.diff( - allocator, - "αβγ", - "αβδ", - false, - ); - defer deinitDiffList(allocator, &greek_diff); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "αβ"), - Diff.init(.delete, "γ"), - Diff.init(.insert, "δ"), - }), greek_diff.items); - } - { - // ө is 0xd3, 0xa9, թ is 0xd6, 0xa9 - var prefix_diff = try this.diff( - allocator, - "abө", - "abթ", - false, - ); - defer deinitDiffList(allocator, &prefix_diff); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "ab"), - Diff.init(.delete, "ө"), - Diff.init(.insert, "թ"), - }), prefix_diff.items); - } - { - var mid_diff = try this.diff( - allocator, - "αөβ", - "αթβ", - false, - ); - defer deinitDiffList(allocator, &mid_diff); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "α"), - Diff.init(.delete, "ө"), - Diff.init(.insert, "թ"), - Diff.init(.equal, "β"), - }), mid_diff.items); - } - { - var mid_prefix = try this.diff( - allocator, - "αβλ", - "αδλ", - false, - ); - defer deinitDiffList(allocator, &mid_prefix); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "α"), - Diff.init(.delete, "β"), - Diff.init(.insert, "δ"), - Diff.init(.equal, "λ"), - }), mid_prefix.items); - } - - var three_prefix = try this.diff( + const dmp = DiffMatchPatch{}; + { + var greek_diff = try dmp.diff( allocator, - "三亥两", - "三亥临", + "αβγ", + "αβδ", false, ); - defer deinitDiffList(allocator, &three_prefix); + defer deinitDiffList(allocator, &greek_diff); try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "三亥"), - Diff.init(.delete, "两"), - Diff.init(.insert, "临"), - }), three_prefix.items); + Diff.init(.equal, "αβ"), + Diff.init(.delete, "γ"), + Diff.init(.insert, "δ"), + }), greek_diff.items); } + { + // ө is 0xd3, 0xa9, թ is 0xd6, 0xa9 + var prefix_diff = try dmp.diff( + allocator, + "abө", + "abթ", + false, + ); + defer deinitDiffList(allocator, &prefix_diff); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "ab"), + Diff.init(.delete, "ө"), + Diff.init(.insert, "թ"), + }), prefix_diff.items); + } + { + var mid_diff = try dmp.diff( + allocator, + "αөβ", + "αթβ", + false, + ); + defer deinitDiffList(allocator, &mid_diff); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "α"), + Diff.init(.delete, "ө"), + Diff.init(.insert, "թ"), + Diff.init(.equal, "β"), + }), mid_diff.items); + } + { + var mid_prefix = try dmp.diff( + allocator, + "αβλ", + "αδλ", + false, + ); + defer deinitDiffList(allocator, &mid_prefix); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "α"), + Diff.init(.delete, "β"), + Diff.init(.insert, "δ"), + Diff.init(.equal, "λ"), + }), mid_prefix.items); + } + + var three_prefix = try dmp.diff( + allocator, + "三亥两", + "三亥临", + false, + ); + defer deinitDiffList(allocator, &three_prefix); + try testing.expectEqualDeep(@as([]const Diff, &.{ + Diff.init(.equal, "三亥"), + Diff.init(.delete, "两"), + Diff.init(.insert, "临"), + }), three_prefix.items); } test "workshop" { @@ -4440,21 +4439,22 @@ test "workshop" { var dmp = DiffMatchPatch{}; dmp.diff_timeout = 0; { - var diffs = try dmp.diff( + var mid_prefix = try dmp.diff( allocator, - "三亥两", - "三亥临", + "αβλ", + "αδλ", false, ); - for (diffs.items) |d| { - std.debug.print("{}\n", .{d}); + defer deinitDiffList(allocator, &mid_prefix); + for (mid_prefix.items) |d| { + std.debug.print("{}", .{d}); } - defer deinitDiffList(allocator, &diffs); try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "三亥"), - Diff.init(.delete, "两"), - Diff.init(.insert, "临"), - }), diffs.items); + Diff.init(.equal, "α"), + Diff.init(.delete, "β"), + Diff.init(.insert, "δ"), + Diff.init(.equal, "λ"), + }), mid_prefix.items); } } From bb91050304da86c9e9f4908910f91590cc58e9ff Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 10:49:50 -0400 Subject: [PATCH 130/176] Restore previous common suffix routine This gets the Greek tests passing, without damaging the initial test of three-byte code point sequences. --- DiffMatchPatch.zig | 39 +++++++++++++-------------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 18d8458..848e522 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -326,25 +326,30 @@ fn diffCommonPrefix(before: []const u8, after: []const u8) usize { fn diffCommonSuffix(before: []const u8, after: []const u8) usize { const n = @min(before.len, after.len); var i: usize = 1; + var was_follow = false; while (i <= n) : (i += 1) { var b = before[before.len - i]; const a = after[after.len - i]; if (a != b) { - // Testing one is fine, because we can only - // have problems if it's both - if (!std.ascii.isASCII(a)) { - if (i == 1) return i - 1; - // Check behind us + if (was_follow) { + // Means we're at at least 2: + assert(i > 1); + // We just saw an identical follow byte, so we back + // out forward: i -= 1; - while (i > 1) : (i -= 1) { + b = before[before.len - i]; + assert(b == after[after.len - i]); + while (i > 1 and is_follow(b)) { + i -= 1; b = before[before.len - i]; assert(b == after[after.len - i]); - if (!is_follow(b)) break; } // Either at one, or no more follow bytes: return i - 1; } else { return i - 1; } + } else { + was_follow = is_follow(b); // no need to check twice } } @@ -850,7 +855,6 @@ fn fixupBisection(text1: []const u8, text2: []const u8, x: usize, y: usize) stru while (y1 != 0) : (y1 -= 1) { if (!is_follow(text2[y1])) break; } - // XXX if (text2[y1] >= 0x80 and y1 != 0) y1 -= 1; } } else { y1 = y; @@ -4436,26 +4440,9 @@ test "Unicode diffs" { test "workshop" { const allocator = std.testing.allocator; + _ = allocator; // autofix var dmp = DiffMatchPatch{}; dmp.diff_timeout = 0; - { - var mid_prefix = try dmp.diff( - allocator, - "αβλ", - "αδλ", - false, - ); - defer deinitDiffList(allocator, &mid_prefix); - for (mid_prefix.items) |d| { - std.debug.print("{}", .{d}); - } - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "α"), - Diff.init(.delete, "β"), - Diff.init(.insert, "δ"), - Diff.init(.equal, "λ"), - }), mid_prefix.items); - } } fn testDiffCleanupSemantic( From ff4e387c1794f5e766675ede7814a8319e33c32b Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 11:45:21 -0400 Subject: [PATCH 131/176] Found a failing test Suspect the cleanup logic in diffCleanupSemantic --- DiffMatchPatch.zig | 103 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 88 insertions(+), 15 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 848e522..403c06c 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -4361,9 +4361,29 @@ test diff { } } +fn diffRoundTrip(allocator: Allocator, dmp: DiffMatchPatch, diff_slice: []const Diff) !void { + var diffs_before = try DiffList.initCapacity(allocator, diff_slice.len); + defer deinitDiffList(allocator, &diffs_before); + for (diff_slice) |item| { + diffs_before.appendAssumeCapacity(.{ .operation = item.operation, .text = try allocator.dupe(u8, item.text) }); + } + const text_before = try diffBeforeText(allocator, diffs_before); + defer allocator.free(text_before); + // XXX std.debug.print("before: {s}\n", .{text_before}); + const text_after = try diffAfterText(allocator, diffs_before); + defer allocator.free(text_after); + // XXX std.debug.print("after: {s}\n", .{text_after}); + var diffs_after = try dmp.diff(allocator, text_before, text_after, false); + defer deinitDiffList(allocator, &diffs_after); + // Should change nothing: + try diffCleanupSemantic(allocator, &diffs_after); + try testing.expectEqualDeep(diffs_before.items, diffs_after.items); +} + test "Unicode diffs" { const allocator = std.testing.allocator; - const dmp = DiffMatchPatch{}; + var dmp = DiffMatchPatch{}; + dmp.diff_timeout = 0; { var greek_diff = try dmp.diff( allocator, @@ -4423,26 +4443,79 @@ test "Unicode diffs" { Diff.init(.equal, "λ"), }), mid_prefix.items); } - - var three_prefix = try dmp.diff( - allocator, - "三亥两", - "三亥临", - false, - ); - defer deinitDiffList(allocator, &three_prefix); - try testing.expectEqualDeep(@as([]const Diff, &.{ - Diff.init(.equal, "三亥"), - Diff.init(.delete, "两"), - Diff.init(.insert, "临"), - }), three_prefix.items); + { // "三亥临" Three-byte, one different suffix + try testing.checkAllAllocationFailures( + allocator, + diffRoundTrip, + .{ + dmp, &.{ + .{ .operation = .equal, .text = "三亥" }, + .{ .operation = .delete, .text = "两" }, + .{ .operation = .insert, .text = "临" }, + }, + }, + ); + } + { // "三亥乤" Three-byte, one middle difference in suffix + try testing.checkAllAllocationFailures( + allocator, + diffRoundTrip, + .{ + dmp, &.{ + .{ .operation = .equal, .text = "三亥" }, + .{ .operation = .delete, .text = "两" }, + .{ .operation = .insert, .text = "乤" }, + }, + }, + ); + } + { // "三亥帤" Three-byte, one prefix difference in suffix + try testing.checkAllAllocationFailures( + allocator, + diffRoundTrip, + .{ + dmp, &.{ + .{ .operation = .equal, .text = "三亥" }, + .{ .operation = .delete, .text = "两" }, + .{ .operation = .insert, .text = "帤" }, + }, + }, + ); + } + { // "三帤亥" Three-byte, one prefix difference in middle + try testing.checkAllAllocationFailures( + allocator, + diffRoundTrip, + .{ + dmp, &.{ + .{ .operation = .equal, .text = "三" }, + .{ .operation = .delete, .text = "两" }, + .{ .operation = .insert, .text = "帤" }, + .{ .operation = .equal, .text = "亥" }, + }, + }, + ); + } } test "workshop" { const allocator = std.testing.allocator; - _ = allocator; // autofix var dmp = DiffMatchPatch{}; dmp.diff_timeout = 0; + { // "三乤亥" Three-byte, one middle difference in middle + try testing.checkAllAllocationFailures( + allocator, + diffRoundTrip, + .{ + dmp, &.{ + .{ .operation = .equal, .text = "三" }, + .{ .operation = .delete, .text = "两" }, + .{ .operation = .insert, .text = "乤" }, + .{ .operation = .equal, .text = "亥" }, + }, + }, + ); + } } fn testDiffCleanupSemantic( From 3f48fd8ee4dee1707fc0a7f25b25d4a4ee8e814f Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 12:15:33 -0400 Subject: [PATCH 132/176] Check for lead byte in common suffix Once I have a comprehensive test set for Unicode problems, I can circle back and try to simplify some of these algorithms. Specifically I don't like that commonSuffix is using a boolean to track the prior status of bytes, it's a test in a hot loop that will actually never trigger with ASCII, which is an important use case to support. Ideally the Unicode code is slower than ASCII only in boundary checks. --- DiffMatchPatch.zig | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 403c06c..c5d5066 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -324,6 +324,9 @@ fn diffCommonPrefix(before: []const u8, after: []const u8) usize { /// Find a common suffix which respects UTF-8 code point boundaries fn diffCommonSuffix(before: []const u8, after: []const u8) usize { + // TODO I don't like the was_follow idiom, I think it's ok to + // just check for non-ascii so we don't need an extra test in + // a hot loop. const n = @min(before.len, after.len); var i: usize = 1; var was_follow = false; @@ -343,6 +346,9 @@ fn diffCommonSuffix(before: []const u8, after: []const u8) usize { i -= 1; b = before[before.len - i]; assert(b == after[after.len - i]); + // TODO why are ASCII and lead bytes different here? + // empirically they are. + if (b > 0xc0) return i; } // Either at one, or no more follow bytes: return i - 1; } else { @@ -4369,10 +4375,10 @@ fn diffRoundTrip(allocator: Allocator, dmp: DiffMatchPatch, diff_slice: []const } const text_before = try diffBeforeText(allocator, diffs_before); defer allocator.free(text_before); - // XXX std.debug.print("before: {s}\n", .{text_before}); + std.debug.print("before: {s}\n", .{text_before}); const text_after = try diffAfterText(allocator, diffs_before); defer allocator.free(text_after); - // XXX std.debug.print("after: {s}\n", .{text_after}); + std.debug.print("after: {s}\n", .{text_after}); var diffs_after = try dmp.diff(allocator, text_before, text_after, false); defer deinitDiffList(allocator, &diffs_after); // Should change nothing: From 27da8d45a3743f9cca7405752970e60765cdf28e Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 12:37:53 -0400 Subject: [PATCH 133/176] All three-byte one-character edits pass This is likely to generalize well, but I'm going to want to check coverage before jumping to that conclusion. --- DiffMatchPatch.zig | 80 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 73 insertions(+), 7 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index c5d5066..efe4ef0 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -348,7 +348,7 @@ fn diffCommonSuffix(before: []const u8, after: []const u8) usize { assert(b == after[after.len - i]); // TODO why are ASCII and lead bytes different here? // empirically they are. - if (b > 0xc0) return i; + if (b > 0xc0) return i; // 0xc0 and 0xc1 are illegal } // Either at one, or no more follow bytes: return i - 1; } else { @@ -4375,10 +4375,10 @@ fn diffRoundTrip(allocator: Allocator, dmp: DiffMatchPatch, diff_slice: []const } const text_before = try diffBeforeText(allocator, diffs_before); defer allocator.free(text_before); - std.debug.print("before: {s}\n", .{text_before}); + // XXX std.debug.print("before: {s}\n", .{text_before}); const text_after = try diffAfterText(allocator, diffs_before); defer allocator.free(text_after); - std.debug.print("after: {s}\n", .{text_after}); + // XXX std.debug.print("after: {s}\n", .{text_after}); var diffs_after = try dmp.diff(allocator, text_before, text_after, false); defer deinitDiffList(allocator, &diffs_after); // Should change nothing: @@ -4502,22 +4502,88 @@ test "Unicode diffs" { }, ); } + { // "三乤亥" Three-byte, one middle difference in middle + try testing.checkAllAllocationFailures( + allocator, + diffRoundTrip, + .{ + dmp, &.{ + .{ .operation = .equal, .text = "三" }, + .{ .operation = .delete, .text = "两" }, + .{ .operation = .insert, .text = "乤" }, + .{ .operation = .equal, .text = "亥" }, + }, + }, + ); + } + { // "三临亥" Three-byte, one suffix difference in middle + try testing.checkAllAllocationFailures( + allocator, + diffRoundTrip, + .{ + dmp, &.{ + .{ .operation = .equal, .text = "三" }, + .{ .operation = .delete, .text = "两" }, + .{ .operation = .insert, .text = "临" }, + .{ .operation = .equal, .text = "亥" }, + }, + }, + ); + } + { // "临三亥" Three-byte, one suffix difference in prefix + try testing.checkAllAllocationFailures( + allocator, + diffRoundTrip, + .{ + dmp, &.{ + .{ .operation = .delete, .text = "两" }, + .{ .operation = .insert, .text = "临" }, + .{ .operation = .equal, .text = "三亥" }, + }, + }, + ); + } + { // "乤三亥" Three-byte, one middle difference in prefix + try testing.checkAllAllocationFailures( + allocator, + diffRoundTrip, + .{ + dmp, &.{ + .{ .operation = .delete, .text = "两" }, + .{ .operation = .insert, .text = "乤" }, + .{ .operation = .equal, .text = "三亥" }, + }, + }, + ); + } + { // "乤三亥" Three-byte, one prefix difference in prefix + try testing.checkAllAllocationFailures( + allocator, + diffRoundTrip, + .{ + dmp, &.{ + .{ .operation = .delete, .text = "两" }, + .{ .operation = .insert, .text = "帤" }, + .{ .operation = .equal, .text = "三亥" }, + }, + }, + ); + } } test "workshop" { const allocator = std.testing.allocator; var dmp = DiffMatchPatch{}; dmp.diff_timeout = 0; - { // "三乤亥" Three-byte, one middle difference in middle + { // "乤三亥" Three-byte, one middle difference in prefix try testing.checkAllAllocationFailures( allocator, diffRoundTrip, .{ dmp, &.{ - .{ .operation = .equal, .text = "三" }, .{ .operation = .delete, .text = "两" }, - .{ .operation = .insert, .text = "乤" }, - .{ .operation = .equal, .text = "亥" }, + .{ .operation = .insert, .text = "帤" }, + .{ .operation = .equal, .text = "三亥" }, }, }, ); From d73e3edefa8d6ecf72b087371ab43c7260bcf495 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 12:54:23 -0400 Subject: [PATCH 134/176] A common-middle test with one permutation To thoroughly test every four-byte permutation, I'm going to have to resort to codegen. --- DiffMatchPatch.zig | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index efe4ef0..8affa90 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1697,7 +1697,8 @@ fn diffCommonOverlap(text1_in: []const u8, text2_in: []const u8) usize { }; if (best_idx == 0) return best_idx; // This would mean a truncation: lead or follow, followed by a follow - // which differs (or it would be included in our overlap) + // which differs (or it would be included in our overlap). + // TODO this currently appears to be dead code, keep an eye on that. if (text2[best_idx] >= 0x80 and is_follow(text2[best_idx + 1])) { // back out assert(best_idx == best); @@ -4569,21 +4570,36 @@ test "Unicode diffs" { }, ); } + { // "三临亥" → "三丿亥" Three-byte, one suffix difference + try testing.checkAllAllocationFailures( + allocator, + diffRoundTrip, + .{ + dmp, &.{ + .{ .operation = .equal, .text = "三" }, + .{ .operation = .delete, .text = "临" }, + .{ .operation = .insert, .text = "丿" }, + .{ .operation = .equal, .text = "亥" }, + }, + }, + ); + } } test "workshop" { const allocator = std.testing.allocator; var dmp = DiffMatchPatch{}; dmp.diff_timeout = 0; - { // "乤三亥" Three-byte, one middle difference in prefix + { // "三临亥" → "三丿亥" Three-byte, one suffix difference try testing.checkAllAllocationFailures( allocator, diffRoundTrip, .{ dmp, &.{ - .{ .operation = .delete, .text = "两" }, - .{ .operation = .insert, .text = "帤" }, - .{ .operation = .equal, .text = "三亥" }, + .{ .operation = .equal, .text = "三" }, + .{ .operation = .delete, .text = "临" }, + .{ .operation = .insert, .text = "丿" }, + .{ .operation = .equal, .text = "亥" }, }, }, ); From 306f1b52aead66857ca47f234ea0f1f8f4caeadb Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 13:24:03 -0400 Subject: [PATCH 135/176] Add basic four-byte tests I don't actually think adding another eight tests to cover the prefixed and suffixed cases will improve coverage in a meaningful way. --- DiffMatchPatch.zig | 81 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 12 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 8affa90..16227ca 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1699,7 +1699,16 @@ fn diffCommonOverlap(text1_in: []const u8, text2_in: []const u8) usize { // This would mean a truncation: lead or follow, followed by a follow // which differs (or it would be included in our overlap). // TODO this currently appears to be dead code, keep an eye on that. + // Reasoning: we're looking for a suffix which matches a prefix, and + // we've already assured that edits end with a follow byte, and begin + // with a lead byte, ASCII being both for our purposes. So a split + // should not be possible. + // I'm going to add a panic just so I know if test cases of any sort + // trigger this code path. if (text2[best_idx] >= 0x80 and is_follow(text2[best_idx + 1])) { + if (true) { + @panic("Your assumption regarding diffCommonOverlap is invalid!"); + } // back out assert(best_idx == best); if (!is_follow(text2[best])) { @@ -4376,10 +4385,8 @@ fn diffRoundTrip(allocator: Allocator, dmp: DiffMatchPatch, diff_slice: []const } const text_before = try diffBeforeText(allocator, diffs_before); defer allocator.free(text_before); - // XXX std.debug.print("before: {s}\n", .{text_before}); const text_after = try diffAfterText(allocator, diffs_before); defer allocator.free(text_after); - // XXX std.debug.print("after: {s}\n", .{text_after}); var diffs_after = try dmp.diff(allocator, text_before, text_after, false); defer deinitDiffList(allocator, &diffs_after); // Should change nothing: @@ -4584,22 +4591,72 @@ test "Unicode diffs" { }, ); } -} - -test "workshop" { - const allocator = std.testing.allocator; - var dmp = DiffMatchPatch{}; - dmp.diff_timeout = 0; - { // "三临亥" → "三丿亥" Three-byte, one suffix difference + { // Four-byte permutation #1 try testing.checkAllAllocationFailures( allocator, diffRoundTrip, .{ dmp, &.{ - .{ .operation = .equal, .text = "三" }, - .{ .operation = .delete, .text = "临" }, + .{ .operation = .equal, .text = "😹💋" }, + .{ .operation = .delete, .text = "\xf0\x9f\xa5\xb9" }, .{ .operation = .insert, .text = "丿" }, - .{ .operation = .equal, .text = "亥" }, + .{ .operation = .equal, .text = "👀🫵" }, + }, + }, + ); + } + { // Four-byte permutation #1 + try testing.checkAllAllocationFailures( + allocator, + diffRoundTrip, + .{ + dmp, &.{ + .{ .operation = .equal, .text = "😹💋" }, + .{ .operation = .delete, .text = "\xf0\x9f\xa5\xb9" }, + .{ .operation = .insert, .text = "\xf1\x9f\xa5\xb9" }, + .{ .operation = .equal, .text = "👀🫵" }, + }, + }, + ); + } + { // Four-byte permutation #2 + try testing.checkAllAllocationFailures( + allocator, + diffRoundTrip, + .{ + dmp, &.{ + .{ .operation = .equal, .text = "😹💋" }, + .{ .operation = .delete, .text = "\xf0\x9f\xa5\xb9" }, + .{ .operation = .insert, .text = "\xf0\xa0\xa5\xb9" }, + .{ .operation = .equal, .text = "👀🫵" }, + }, + }, + ); + } + { // Four-byte permutation #3 + try testing.checkAllAllocationFailures( + allocator, + diffRoundTrip, + .{ + dmp, &.{ + .{ .operation = .equal, .text = "😹💋" }, + .{ .operation = .delete, .text = "\xf0\x9f\xa5\xb9" }, + .{ .operation = .insert, .text = "\xf0\x9f\xa4\xb9" }, + .{ .operation = .equal, .text = "👀🫵" }, + }, + }, + ); + } + { // Four-byte permutation #4 + try testing.checkAllAllocationFailures( + allocator, + diffRoundTrip, + .{ + dmp, &.{ + .{ .operation = .equal, .text = "😹💋" }, + .{ .operation = .delete, .text = "\xf0\x9f\xa5\xb9" }, + .{ .operation = .insert, .text = "\xf0\x9f\xa5\xb4" }, + .{ .operation = .equal, .text = "👀🫵" }, }, }, ); From 4053973c26ec2f73b7ccea1c789efcbe63a7ad05 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 13:44:13 -0400 Subject: [PATCH 136/176] Doc comment for diffRoundTrip --- DiffMatchPatch.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 16227ca..043a9f6 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -4377,6 +4377,7 @@ test diff { } } +/// Round-trip a diff, confirming that the result matches the original. fn diffRoundTrip(allocator: Allocator, dmp: DiffMatchPatch, diff_slice: []const Diff) !void { var diffs_before = try DiffList.initCapacity(allocator, diff_slice.len); defer deinitDiffList(allocator, &diffs_before); From 53167e15018cb40fd99c7281041da454bd15e641 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 13:55:47 -0400 Subject: [PATCH 137/176] Port streamlined boolInt from other branch --- DiffMatchPatch.zig | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 043a9f6..a2f8ead 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1550,11 +1550,8 @@ fn diffCleanupSemanticScore(one: []const u8, two: []const u8) usize { return 0; } -inline fn boolInt(b: bool) usize { - if (b) - return 1 - else - return 0; +inline fn boolInt(b: bool) u8 { + return @intFromBool(b); } /// Reduce the number of edits by eliminating operationally trivial From fe5e9a7c39fe0bc4edf118f0adbe43d84bd592f2 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 16:09:20 -0400 Subject: [PATCH 138/176] Simplify and correct patch adjustment This uses a very simple while loop, and I see no reason why adjusting a diff boundary would need to be more complex than this in other cases. --- DiffMatchPatch.zig | 14 +++----------- build.zig | 28 ++++++++++++++++++---------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index a2f8ead..ec4c60f 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2340,9 +2340,8 @@ fn patchAddContext( const prefix = pre: { var pre_start = if (padding > patch.start2) 0 else patch.start2 - padding; // Make sure we're not breaking a codepoint. - while (is_follow(text[pre_start]) and pre_start > 0) { - pre_start -= 1; - } // Assuming we did everything else right, pre_end should be + while (is_follow(text[pre_start]) and pre_start > 0) : (pre_start -= 1) {} + // Assuming we did everything else right, pre_end should be // properly placed. break :pre text[pre_start..patch.start2]; }; @@ -2358,14 +2357,7 @@ fn patchAddContext( const post_start = patch.start2 + patch.length1; var post_end = @min(text.len, patch.start2 + patch.length1 + padding); // Prevent broken codepoints here as well: Lead bytes, or follow with another follow - while (post_end + 1 < text.len and !std.ascii.isASCII(text[post_end]) and is_follow(text[post_end + 1])) { - post_end += 1; - // Special case: penultimate with another follow at end - if (post_end + 2 == text.len and is_follow(text[post_end + 1])) { - post_end += 1; - break; // Not actually necessary, but polite. - } - } + while (post_end < text.len and is_follow(text[post_end])) : (post_end += 1) {} post_end = @min(post_end, text.len); break :post text[post_start..post_end]; }; diff --git a/build.zig b/build.zig index c29c143..39db8fb 100644 --- a/build.zig +++ b/build.zig @@ -18,22 +18,30 @@ pub fn build(b: *std.Build) void { .optimize = optimize, }); const step_tests = b.addRunArtifact(tests); - step_tests.has_side_effects = true; b.step("test", "Run diffz tests").dependOn(&step_tests.step); - // Adds a step to generate code coverage - const cov_step = b.step("cov", "Generate coverage (kcov must be installed)"); + const addOutputDirectoryArg = comptime if (@import("builtin").zig_version.order(.{ .major = 0, .minor = 13, .patch = 0 }) == .lt) + std.Build.Step.Run.addOutputFileArg + else + std.Build.Step.Run.addOutputDirectoryArg; - const cov_run = b.addSystemCommand(&.{ + const run_kcov = b.addSystemCommand(&.{ "kcov", "--clean", - "--include-pattern=DiffMatchPatch.zig", "--exclude-line=unreachable,expect(false)", - "kcov-output", }); - cov_run.addArtifactArg(tests); - cov_step.dependOn(&cov_run.step); - _ = cov_run.captureStdOut(); - _ = cov_run.captureStdErr(); + run_kcov.addPrefixedDirectoryArg("--include-pattern=", b.path(".")); + const coverage_output = addOutputDirectoryArg(run_kcov, "."); + run_kcov.addArtifactArg(tests); + run_kcov.enableTestRunnerMode(); + + const install_coverage = b.addInstallDirectory(.{ + .source_dir = coverage_output, + .install_dir = .{ .custom = "coverage" }, + .install_subdir = "", + }); + + const coverage_step = b.step("coverage", "Generate coverage (kcov must be installed)"); + coverage_step.dependOn(&install_coverage.step); } From b4e8913f9d7c0a5f7e581130868bc851c6183e7d Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 17:04:44 -0400 Subject: [PATCH 139/176] Simplify split-fix logic for diffBisectSplit --- DiffMatchPatch.zig | 64 ++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index ec4c60f..113281a 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -819,7 +819,8 @@ fn diffBisectSplit( y: isize, deadline: u64, ) DiffError!DiffList { - const x1, const y1 = fixupBisection(text1, text2, @intCast(x), @intCast(y)); + const x1 = fixSplitForward(text1, @intCast(x)); + const y1 = fixSplitBackward(text2, @intCast(y)); const text1a = text1[0..x1]; const text2a = text2[0..y1]; const text1b = text1[x1..]; @@ -840,32 +841,16 @@ fn diffBisectSplit( return diffs; } -/// Fix Unicode clipping problems with bisection points. -/// Moves text1 forward and text2 backward, in case they split on the same point. -fn fixupBisection(text1: []const u8, text2: []const u8, x: usize, y: usize) struct { usize, usize } { - var x1: usize = undefined; - var y1: usize = undefined; - if (x < text1.len and is_follow(text1[x])) { - x1 = x + 1; - if (x1 != text1.len) { - while (x1 < text1.len) : (x1 += 1) { - if (!is_follow(text1[x1])) break; - } - } - } else { - x1 = x; - } - if (y < text2.len and is_follow(text2[y])) { - y1 = y - 1; - if (y1 != 0) { - while (y1 != 0) : (y1 -= 1) { - if (!is_follow(text2[y1])) break; - } - } - } else { - y1 = y; - } - return .{ x1, y1 }; +inline fn fixSplitForward(text: []const u8, i: usize) usize { + var idx = i; + while (idx < text.len and is_follow(text[idx])) : (idx += 1) {} + return idx; +} + +inline fn fixSplitBackward(text: []const u8, i: usize) usize { + var idx = i; + if (idx < text.len) while (idx != 0 and is_follow(text[idx])) : (idx -= 1) {}; + return idx; } /// Do a quick line-level diff on both strings, then rediff the parts for @@ -4653,6 +4638,14 @@ test "Unicode diffs" { } } +test "Diff format" { + const a_diff = Diff{ .operation = .insert, .text = "add me" }; + const expect = "(+, \"add me\")"; + var out_buf: [13]u8 = undefined; + const out_string = try std.fmt.bufPrint(&out_buf, "{}", .{a_diff}); + try testing.expectEqualStrings(expect, out_string); +} + fn testDiffCleanupSemantic( allocator: std.mem.Allocator, params: struct { @@ -5507,7 +5500,22 @@ test "testPatchAddContext" { "@@ -1,27 +1,28 @@\n Th\n-e\n+at\n quick brown fox jumps. \n", }, ); - // TODO: This will need some patches which check the Unicode handling. + // Unicode + try std.testing.checkAllAllocationFailures( + allocator, + testPatchAddContext, + .{ + dmp, + "@@ -9,6 +10,3 @@\n-remove\n+add\n", + "⊗⊘⊙remove⊙⊘⊗", + \\@@ -3,18 +4,15 @@ + \\ %E2%8A%98%E2%8A%99 + \\-remove + \\+add + \\ %E2%8A%99%E2%8A%98 + \\ + }, + ); } fn testMakePatch(allocator: Allocator) !void { From 7b1822034ff849cbb5d2ea386f951f6eb94e9798 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 17:12:06 -0400 Subject: [PATCH 140/176] Simplify common prefix logic with fixSplitBackward The suffix case turns out to be more complex. I suspect that it can be simplified more than it is, however. --- DiffMatchPatch.zig | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 113281a..85f8baf 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -299,23 +299,10 @@ fn diffCommonPrefix(before: []const u8, after: []const u8) usize { const n = @min(before.len, after.len); var i: usize = 0; while (i < n) : (i += 1) { - var b = before[i]; + const b = before[i]; const a = after[i]; if (a != b) { - if (is_follow(a) and is_follow(b)) { - // We've clipped a codepoint, back out - if (i == 0) return i; // Malformed UTF-8 is always possible - i -= 1; - while (i != 0) : (i -= 1) { - b = before[i]; - assert(b == after[i]); - if (!is_follow(b)) break; - } - // Now we're either at zero, or at the lead, - return i; - } else { - return i; - } + return fixSplitBackward(before, i); } } From 7e7e5e97f4781f380aab14070619c5ba8e9c1127 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 17:42:15 -0400 Subject: [PATCH 141/176] Get rid of was_follow check in commonSuffix I think it's relatively straightforward from here. --- DiffMatchPatch.zig | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 85f8baf..61a2ae2 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -311,17 +311,14 @@ fn diffCommonPrefix(before: []const u8, after: []const u8) usize { /// Find a common suffix which respects UTF-8 code point boundaries fn diffCommonSuffix(before: []const u8, after: []const u8) usize { - // TODO I don't like the was_follow idiom, I think it's ok to - // just check for non-ascii so we don't need an extra test in - // a hot loop. const n = @min(before.len, after.len); var i: usize = 1; - var was_follow = false; while (i <= n) : (i += 1) { var b = before[before.len - i]; const a = after[after.len - i]; if (a != b) { - if (was_follow) { + if (i == 1) return 0; + if (is_follow(before[before.len - i + 1])) { // Means we're at at least 2: assert(i > 1); // We just saw an identical follow byte, so we back @@ -341,8 +338,6 @@ fn diffCommonSuffix(before: []const u8, after: []const u8) usize { } else { return i - 1; } - } else { - was_follow = is_follow(b); // no need to check twice } } From 6fc6c89f2c17fe77baa95b76357897b9410fb4a7 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 17:44:52 -0400 Subject: [PATCH 142/176] Replace code in commonSuffix with fixSplitForward I knew there was a way to do it, just had to figure out how it all fits together backward. --- DiffMatchPatch.zig | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 61a2ae2..408202b 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -314,30 +314,11 @@ fn diffCommonSuffix(before: []const u8, after: []const u8) usize { const n = @min(before.len, after.len); var i: usize = 1; while (i <= n) : (i += 1) { - var b = before[before.len - i]; + const b = before[before.len - i]; const a = after[after.len - i]; if (a != b) { if (i == 1) return 0; - if (is_follow(before[before.len - i + 1])) { - // Means we're at at least 2: - assert(i > 1); - // We just saw an identical follow byte, so we back - // out forward: - i -= 1; - b = before[before.len - i]; - assert(b == after[after.len - i]); - while (i > 1 and is_follow(b)) { - i -= 1; - b = before[before.len - i]; - assert(b == after[after.len - i]); - // TODO why are ASCII and lead bytes different here? - // empirically they are. - if (b > 0xc0) return i; // 0xc0 and 0xc1 are illegal - } // Either at one, or no more follow bytes: - return i - 1; - } else { - return i - 1; - } + return before.len - fixSplitForward(before, before.len - i + 1); } } From de4ef8d10b0c1e234fab9be39f850fa08257bb41 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 17:48:46 -0400 Subject: [PATCH 143/176] Use fixSplits in patchAddContext Now the action is all isolated in a single source of truth. --- DiffMatchPatch.zig | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 408202b..f97bbc8 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2288,7 +2288,7 @@ fn patchAddContext( const prefix = pre: { var pre_start = if (padding > patch.start2) 0 else patch.start2 - padding; // Make sure we're not breaking a codepoint. - while (is_follow(text[pre_start]) and pre_start > 0) : (pre_start -= 1) {} + pre_start = fixSplitBackward(text, pre_start); // Assuming we did everything else right, pre_end should be // properly placed. break :pre text[pre_start..patch.start2]; @@ -2304,9 +2304,8 @@ fn patchAddContext( const suffix = post: { const post_start = patch.start2 + patch.length1; var post_end = @min(text.len, patch.start2 + patch.length1 + padding); - // Prevent broken codepoints here as well: Lead bytes, or follow with another follow - while (post_end < text.len and is_follow(text[post_end])) : (post_end += 1) {} - post_end = @min(post_end, text.len); + // Prevent broken codepoints here as well + post_end = fixSplitForward(text, post_end); break :post text[post_start..post_end]; }; if (suffix.len != 0) { From 85bd8968573ca2722ac6683534a55ef87a88c3d0 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 20:52:41 -0400 Subject: [PATCH 144/176] Adds revamp of diffLine mode Untested but ready for deployment. In addition to using codepoints, this refactoring has a function taking an iterator, which returns the lines. This opens up the ability to stream from any iterator which the program calls for. This can include a tokenizer, for sharp semantic diffing of source code files. --- DiffMatchPatch.zig | 130 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 129 insertions(+), 1 deletion(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index f97bbc8..b1a0fab 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -964,7 +964,6 @@ fn diffLinesToCharsMunge( var chars = ArrayListUnmanaged(u8){}; defer chars.deinit(allocator); // Walk the text, pulling out a Substring for each line. - // TODO this can be handled with a Reader, avoiding all the manual splitting while (line_end < @as(isize, @intCast(text.len)) - 1) { line_end = b: { break :b @as(isize, @intCast(std.mem.indexOf(u8, text[@intCast(line_start)..], "\n") orelse @@ -1011,6 +1010,135 @@ fn diffCharsToLines( } } +/// Split a text into a list of strings. Reduce the texts to a string of +/// hashes where each Unicode character represents one line. +/// @param text String to encode. +/// @param lineArray List of unique strings. +/// @param lineHash Map of strings to indices. +/// @param maxLines Maximum length of lineArray. +/// @return Encoded string. +fn diffLinesToCharsMunge2( + allocator: std.mem.Allocator, + text: []const u8, + line_array: *ArrayListUnmanaged([]const u8), + line_hash: *std.StringHashMapUnmanaged(usize), + max_lines: usize, +) DiffError![]const u8 { + var iter = LineIterator{ .text = text }; + return try diffIteratorToCharsMunge( + allocator, + text, + &line_array, + &line_hash, + &iter, + max_lines, + ); +} + +/// Split a text into segments. Reduce the texts to a string of +/// hashes where each Unicode character represents one segment. +/// @param text String to encode. +/// @param segment_array List of unique string segments. +/// @param line_hash Map of strings to indices into segment_array. +/// @param iterator Returns the next segment. Must have functions +/// next(), returning the next segment, and short_circuit(), +/// called when max_segments is reached. +/// @param max_segments Maximum length of lineArray. +/// @return Encoded string. +fn diffIteratorToCharsMunge( + allocator: std.mem.Allocator, + text: []const u8, + segment_array: *ArrayListUnmanaged([]const u8), + segment_hash: *std.StringHashMapUnmanaged(u21), + iterator: anytype, + max_segments: usize, +) DiffError![]const u8 { + var chars = ArrayListUnmanaged(u8){}; + defer chars.deinit(allocator); + var count = 0; + var codepoint: u21 = 32; + var char_buf: [4]u8 = undefined; + while (iterator.next()) |line| { + if (segment_hash.get(line)) |value| { + const nbytes = try std.unicode.wtf8Encode(value, &char_buf) catch unreachable; + try chars.appendSlice(allocator, char_buf[0..nbytes]); + count += line.len; + } else { + if (codepoint - 32 == max_segments) { + // Bail out + iterator.short_circuit(); + const final_line = text[count..]; + try segment_array.append(allocator, final_line); + try segment_hash.put(allocator, final_line, codepoint); + const nbytes = try std.unicode.wtf8Encode(codepoint, &char_buf) catch unreachable; + try chars.appendSlice(allocator, char_buf[0..nbytes]); + } + try segment_array.append(allocator, line); + try segment_hash.put(allocator, line, codepoint); + const nbytes = try std.unicode.wtf8Encode(codepoint, &char_buf) catch unreachable; + try chars.appendSlice(allocator, char_buf[0..nbytes]); + codepoint += 1; + } + } +} + +/// Rehydrate the text in a diff from a string of line hashes to real lines +/// of text. +/// @param diffs List of Diff objects. +/// @param lineArray List of unique strings. +fn diffCharsToLines2( + allocator: Allocator, + diffs: []Diff, + line_array: []const []const u8, +) DiffError!void { + var text = ArrayListUnmanaged(u8){}; + defer text.deinit(allocator); + for (diffs) |*d| { + var cursor: usize = 0; + while (cursor < d.text.len) { + const cp_len = std.unicode.utf8ByteSequenceLength(text[cursor]) catch { + @panic("Internal decode error in diffsCharsToLines"); + }; + const cp = try std.unicode.wtf8Decode(text[cursor..][0..cp_len]) catch { + @panic("Internal decode error in diffCharsToLines"); + }; + try text.appendSlice(line_array[cp - 32]); + cursor += cp_len; + } + allocator.free(d.text); + d.text = try text.toOwnedSlice(); + } +} + +/// An iteration struct over lines, which includes the newline if present. +const LineIterator = struct { + cursor: usize = 0, + text: []const u8, + + pub fn next(iter: *LineIterator) ?[]const u8 { + if (iter.cursor == iter.text.len) return null; + const maybe_newline = std.mem.indexOfPos( + u8, + iter.text, + iter.cursor, + '\n', + ); + if (maybe_newline) |nl| { + const line = iter.text[iter.cursor .. nl + 1]; + iter.cursor = nl + 1; + return line; + } else { + const line = iter.text[iter.cursor..]; + iter.cursor = iter.text.len; + return line; + } + } + + pub fn short_circuit(iter: *LineIterator) void { + iter.cursor = iter.text.len; + } +}; + /// Reorder and merge like edit sections. Merge equalities. /// Any edit section can move as long as it doesn't cross an equality. /// @param diffs List of Diff objects. From ba2840f5ca1530025c778f6ec0e2379e28f81ea9 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Thu, 11 Jul 2024 21:13:30 -0400 Subject: [PATCH 145/176] Remove unneeded conditional in diffCommonSuffix --- DiffMatchPatch.zig | 2 -- 1 file changed, 2 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index b1a0fab..8efcf69 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -14,7 +14,6 @@ pub const DiffError = error{ }; //| XXX This boolean is entirely for calming the compiler down while working - const XXX = false; //| Fields @@ -317,7 +316,6 @@ fn diffCommonSuffix(before: []const u8, after: []const u8) usize { const b = before[before.len - i]; const a = after[after.len - i]; if (a != b) { - if (i == 1) return 0; return before.len - fixSplitForward(before, before.len - i + 1); } } From 583618be156641e7e90a10ed6b5bdf05884c5e4d Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 11:02:48 -0400 Subject: [PATCH 146/176] Avoid extra allocation for halfMatch The only case where we can borrow it directly is for common_middle. --- DiffMatchPatch.zig | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 8efcf69..dedf48d 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -409,7 +409,8 @@ fn diffCompute( } // Check to see if the problem can be split in two. - if (try dmp.diffHalfMatch(allocator, before, after)) |half_match| { + var maybe_half_match = try dmp.diffHalfMatch(allocator, before, after); + if (maybe_half_match) |*half_match| { // A half-match was found, sort out the return data. defer half_match.deinit(allocator); // Send both pairs off for separate processing. @@ -440,11 +441,9 @@ fn diffCompute( // Merge the results. try diffs.ensureUnusedCapacity(allocator, 1); diffs.appendAssumeCapacity( - Diff.init(.equal, try allocator.dupe( - u8, - half_match.common_middle, - )), + Diff.init(.equal, half_match.common_middle), ); + half_match.common_middle = ""; try diffs.appendSlice(allocator, diffs_b.items); return diffs; } @@ -462,14 +461,13 @@ const HalfMatchResult = struct { suffix_after: []const u8, common_middle: []const u8, - // TODO maybe check for empty slice here for fewer copies, - // as in, maybe we can transfer ownership and replace with "". - pub fn deinit(hmr: HalfMatchResult, alloc: Allocator) void { - alloc.free(hmr.prefix_before); - alloc.free(hmr.suffix_before); - alloc.free(hmr.prefix_after); - alloc.free(hmr.suffix_after); - alloc.free(hmr.common_middle); + // Free the HalfMatchResult's memory. + pub fn deinit(hmr: HalfMatchResult, allocator: Allocator) void { + allocator.free(hmr.prefix_before); + allocator.free(hmr.suffix_before); + allocator.free(hmr.prefix_after); + allocator.free(hmr.suffix_after); + allocator.free(hmr.common_middle); } }; From a785737ed6b3f7ef5c57b109de40552cef48c849 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 11:13:51 -0400 Subject: [PATCH 147/176] Finish adding 2.0 of line mode This way I can keep the tests passing while I port them over. A few of them will no longer be valid, but equivalents are available. --- DiffMatchPatch.zig | 117 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index dedf48d..0af020b 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -597,6 +597,7 @@ fn diffHalfMatchInternal( errdefer allocator.free(prefix_after); const suffix_after = try allocator.dupe(u8, best_short_text_b); const best_common_text = try best_common.toOwnedSlice(allocator); + errdefer allocator.free(best_common_text); // Keeps the code portable. return .{ .prefix_before = prefix_before, .suffix_before = suffix_before, @@ -1006,6 +1007,122 @@ fn diffCharsToLines( } } +/// Do a quick line-level diff on both strings, then rediff the parts for +/// greater accuracy. +/// This speedup can produce non-minimal diffs. +/// @param text1 Old string to be diffed. +/// @param text2 New string to be diffed. +/// @param deadline Time when the diff should be complete by. +/// @return List of Diff objects. +fn diffLineMode2( + dmp: DiffMatchPatch, + allocator: std.mem.Allocator, + text1_in: []const u8, + text2_in: []const u8, + deadline: u64, +) DiffError!DiffList { + // Scan the text on a line-by-line basis first. + var a = try diffLinesToChars2(allocator, text1_in, text2_in); + defer a.deinit(allocator); + const text1 = a.chars_1; + const text2 = a.chars_2; + const line_array = a.line_array; + + var diffs: DiffList = try dmp.diffInternal(allocator, text1, text2, false, deadline); + errdefer diffs.deinit(allocator); + // Convert the diff back to original text. + try diffCharsToLines2(allocator, diffs.items, line_array.items); + // Eliminate freak matches (e.g. blank lines) + try diffCleanupSemantic(allocator, &diffs); + + // Rediff any replacement blocks, this time character-by-character. + // Add a dummy entry at the end. + try diffs.append(allocator, Diff.init(.equal, "")); + + var pointer: usize = 0; + var count_delete: usize = 0; + var count_insert: usize = 0; + var text_delete = ArrayListUnmanaged(u8){}; + var text_insert = ArrayListUnmanaged(u8){}; + defer { + text_delete.deinit(allocator); + text_insert.deinit(allocator); + } + + while (pointer < diffs.items.len) { + switch (diffs.items[pointer].operation) { + .insert => { + count_insert += 1; + try text_insert.appendSlice(allocator, diffs.items[pointer].text); + }, + .delete => { + count_delete += 1; + try text_delete.appendSlice(allocator, diffs.items[pointer].text); + }, + .equal => { + // Upon reaching an equality, check for prior redundancies. + if (count_delete >= 1 and count_insert >= 1) { + // Delete the offending records and add the merged ones. + freeRangeDiffList( + allocator, + &diffs, + pointer - count_delete - count_insert, + count_delete + count_insert, + ); + try diffs.replaceRange( + allocator, + pointer - count_delete - count_insert, + count_delete + count_insert, + &.{}, + ); + pointer = pointer - count_delete - count_insert; + var sub_diff = try dmp.diffInternal(allocator, text_delete.items, text_insert.items, false, deadline); + defer sub_diff.deinit(allocator); + try diffs.insertSlice(allocator, pointer, sub_diff.items); + pointer = pointer + sub_diff.items.len; + } + count_insert = 0; + count_delete = 0; + text_delete.items.len = 0; + text_insert.items.len = 0; + }, + } + pointer += 1; + } + diffs.items.len -= 1; // Remove the dummy entry at the end. + + return diffs; +} + +/// Split two texts into a list of strings. Reduce the texts to a string of +/// hashes where each Unicode character represents one line. +/// @param text1 First string. +/// @param text2 Second string. +/// @return Three element Object array, containing the encoded text1, the +/// encoded text2 and the List of unique strings. The zeroth element +/// of the List of unique strings is intentionally blank. +fn diffLinesToChars2( + allocator: std.mem.Allocator, + text1: []const u8, + text2: []const u8, +) DiffError!LinesToCharsResult { + var line_array = ArrayListUnmanaged([]const u8){}; + errdefer line_array.deinit(allocator); + var line_hash = std.StringHashMapUnmanaged(usize){}; + defer line_hash.deinit(allocator); + // e.g. line_array[4] == "Hello\n" + // e.g. line_hash.get("Hello\n") == 4 + + // "\x00" is a valid character, but various debuggers don't like it. + // So we'll insert a junk entry to avoid generating a null character. + try line_array.append(allocator, ""); + + // Allocate 2/3rds of the space for text1, the rest for text2. + const chars1 = try diffLinesToCharsMunge2(allocator, text1, &line_array, &line_hash, 170); + const chars2 = try diffLinesToCharsMunge2(allocator, text2, &line_array, &line_hash, 255); + return .{ .chars_1 = chars1, .chars_2 = chars2, .line_array = line_array }; +} + /// Split a text into a list of strings. Reduce the texts to a string of /// hashes where each Unicode character represents one line. /// @param text String to encode. From c1d9b536e5aa0480ff6e253b0c4d00da36193987 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 12:07:47 -0400 Subject: [PATCH 148/176] First tests pass. --- DiffMatchPatch.zig | 44 +++++++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 0af020b..11e7e72 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1108,15 +1108,11 @@ fn diffLinesToChars2( ) DiffError!LinesToCharsResult { var line_array = ArrayListUnmanaged([]const u8){}; errdefer line_array.deinit(allocator); - var line_hash = std.StringHashMapUnmanaged(usize){}; + var line_hash = std.StringHashMapUnmanaged(u21){}; defer line_hash.deinit(allocator); // e.g. line_array[4] == "Hello\n" // e.g. line_hash.get("Hello\n") == 4 - // "\x00" is a valid character, but various debuggers don't like it. - // So we'll insert a junk entry to avoid generating a null character. - try line_array.append(allocator, ""); - // Allocate 2/3rds of the space for text1, the rest for text2. const chars1 = try diffLinesToCharsMunge2(allocator, text1, &line_array, &line_hash, 170); const chars2 = try diffLinesToCharsMunge2(allocator, text2, &line_array, &line_hash, 255); @@ -1134,15 +1130,15 @@ fn diffLinesToCharsMunge2( allocator: std.mem.Allocator, text: []const u8, line_array: *ArrayListUnmanaged([]const u8), - line_hash: *std.StringHashMapUnmanaged(usize), + line_hash: *std.StringHashMapUnmanaged(u21), max_lines: usize, ) DiffError![]const u8 { var iter = LineIterator{ .text = text }; return try diffIteratorToCharsMunge( allocator, text, - &line_array, - &line_hash, + line_array, + line_hash, &iter, max_lines, ); @@ -1166,14 +1162,16 @@ fn diffIteratorToCharsMunge( iterator: anytype, max_segments: usize, ) DiffError![]const u8 { + // This makes the unreachables in the function legitimate: + assert(max_segments <= 0x10ffff); // Maximum Unicode codepoint value. var chars = ArrayListUnmanaged(u8){}; defer chars.deinit(allocator); - var count = 0; + var count: usize = 0; var codepoint: u21 = 32; var char_buf: [4]u8 = undefined; while (iterator.next()) |line| { if (segment_hash.get(line)) |value| { - const nbytes = try std.unicode.wtf8Encode(value, &char_buf) catch unreachable; + const nbytes = std.unicode.wtf8Encode(value, &char_buf) catch unreachable; try chars.appendSlice(allocator, char_buf[0..nbytes]); count += line.len; } else { @@ -1183,16 +1181,17 @@ fn diffIteratorToCharsMunge( const final_line = text[count..]; try segment_array.append(allocator, final_line); try segment_hash.put(allocator, final_line, codepoint); - const nbytes = try std.unicode.wtf8Encode(codepoint, &char_buf) catch unreachable; + const nbytes = std.unicode.wtf8Encode(codepoint, &char_buf) catch unreachable; try chars.appendSlice(allocator, char_buf[0..nbytes]); } try segment_array.append(allocator, line); try segment_hash.put(allocator, line, codepoint); - const nbytes = try std.unicode.wtf8Encode(codepoint, &char_buf) catch unreachable; + const nbytes = std.unicode.wtf8Encode(codepoint, &char_buf) catch unreachable; try chars.appendSlice(allocator, char_buf[0..nbytes]); codepoint += 1; } } + return try chars.toOwnedSlice(allocator); } /// Rehydrate the text in a diff from a string of line hashes to real lines @@ -1230,7 +1229,7 @@ const LineIterator = struct { pub fn next(iter: *LineIterator) ?[]const u8 { if (iter.cursor == iter.text.len) return null; - const maybe_newline = std.mem.indexOfPos( + const maybe_newline = std.mem.indexOfScalarPos( u8, iter.text, iter.cursor, @@ -3479,6 +3478,10 @@ fn encodeUri(allocator: std.mem.Allocator, text: []const u8) ![]u8 { return charlist.toOwnedSlice(); } +//| +//| TESTS +//| + test encodeUri { const allocator = std.testing.allocator; const special_chars = "!#$&'()*+,-./:;=?@_~"; @@ -3690,6 +3693,21 @@ test diffHalfMatch { }}); } +test "diffLinesToChars2" { + const allocator = testing.allocator; + // Convert lines down to characters. + var tmp_array_list = std.ArrayList([]const u8).init(allocator); + defer tmp_array_list.deinit(); + try tmp_array_list.append("alpha\n"); + try tmp_array_list.append("beta\n"); + + var result = try diffLinesToChars2(allocator, "alpha\nbeta\nalpha\n", "beta\nalpha\nbeta\n"); + defer result.deinit(allocator); + try testing.expectEqualStrings(" ! ", result.chars_1); // Shared lines #1 + try testing.expectEqualStrings("! !", result.chars_2); // Shared lines #2 + try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // Shared lines #3 +} + test diffLinesToChars { const allocator = testing.allocator; // Convert lines down to characters. From e043eabde76039b609743ba6d3f4d679f26b6e52 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 12:20:09 -0400 Subject: [PATCH 149/176] Track codepoint count between calls --- DiffMatchPatch.zig | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 11e7e72..7c555f1 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1167,7 +1167,7 @@ fn diffIteratorToCharsMunge( var chars = ArrayListUnmanaged(u8){}; defer chars.deinit(allocator); var count: usize = 0; - var codepoint: u21 = 32; + var codepoint: u21 = 32 + cast(u21, segment_array.items.len); var char_buf: [4]u8 = undefined; while (iterator.next()) |line| { if (segment_hash.get(line)) |value| { @@ -3702,10 +3702,46 @@ test "diffLinesToChars2" { try tmp_array_list.append("beta\n"); var result = try diffLinesToChars2(allocator, "alpha\nbeta\nalpha\n", "beta\nalpha\nbeta\n"); - defer result.deinit(allocator); + try testing.expectEqualStrings(" ! ", result.chars_1); // Shared lines #1 try testing.expectEqualStrings("! !", result.chars_2); // Shared lines #2 try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // Shared lines #3 + result.deinit(allocator); + + tmp_array_list.items.len = 0; + try tmp_array_list.append("alpha\r\n"); + try tmp_array_list.append("beta\r\n"); + try tmp_array_list.append("\r\n"); + + result = try diffLinesToChars2(allocator, "", "alpha\r\nbeta\r\n\r\n\r\n"); + try testing.expectEqualStrings("", result.chars_1); // Empty string and blank lines #1 + try testing.expectEqualStrings(" !\"\"", result.chars_2); // Empty string and blank lines #2 + try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // Empty string and blank lines #3 + result.deinit(allocator); + tmp_array_list.items.len = 0; + try tmp_array_list.append("a"); + try tmp_array_list.append("b"); + + result = try diffLinesToChars2(allocator, "a", "b"); + try testing.expectEqualStrings(" ", result.chars_1); // No linebreaks #1. + try testing.expectEqualStrings("!", result.chars_2); // No linebreaks #2. + try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // No linebreaks #3. + result.deinit(allocator); +} + +test "workshop" { + const allocator = testing.allocator; + // Convert lines down to characters. + var tmp_array_list = std.ArrayList([]const u8).init(allocator); + defer tmp_array_list.deinit(); + try tmp_array_list.append("a"); + try tmp_array_list.append("b"); + + var result = try diffLinesToChars2(allocator, "a", "b"); + try testing.expectEqualStrings(" ", result.chars_1); // No linebreaks #1. + try testing.expectEqualStrings("!", result.chars_2); // No linebreaks #2. + try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // No linebreaks #3. + result.deinit(allocator); } test diffLinesToChars { @@ -3733,7 +3769,7 @@ test diffLinesToChars { try testing.expectEqualStrings("", result.chars_1); // Empty string and blank lines #1 try testing.expectEqualStrings("\u{0001}\u{0002}\u{0003}\u{0003}", result.chars_2); // Empty string and blank lines #2 try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // Empty string and blank lines #3 - + // ------ tmp_array_list.items.len = 0; try tmp_array_list.append(""); try tmp_array_list.append("a"); From 780d3e9428736e7b8a4308a61d9ab9977c4eff1a Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 12:58:50 -0400 Subject: [PATCH 150/176] Ports basic diffLinesToChars tests The interesting part is if it sorts lines correctly. --- DiffMatchPatch.zig | 64 +++++++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 7c555f1..b5831e3 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1094,6 +1094,16 @@ fn diffLineMode2( return diffs; } +// These numbers have a 32 point buffer, to avoid annoyance with +// c0 control characters. The algorithm drops the bottom points, +// not the top, that is, it will use 0x10ffff given enough unique +// lines. +const UNICODE_MAX = 0x0010ffdf; +const UNICODE_TWO_THIRDS = 742724; +const UNICODE_ONE_THIRD = 371355; +comptime { + assert(UNICODE_TWO_THIRDS + UNICODE_ONE_THIRD == UNICODE_MAX); +} /// Split two texts into a list of strings. Reduce the texts to a string of /// hashes where each Unicode character represents one line. /// @param text1 First string. @@ -1114,8 +1124,8 @@ fn diffLinesToChars2( // e.g. line_hash.get("Hello\n") == 4 // Allocate 2/3rds of the space for text1, the rest for text2. - const chars1 = try diffLinesToCharsMunge2(allocator, text1, &line_array, &line_hash, 170); - const chars2 = try diffLinesToCharsMunge2(allocator, text2, &line_array, &line_hash, 255); + const chars1 = try diffLinesToCharsMunge2(allocator, text1, &line_array, &line_hash, UNICODE_TWO_THIRDS); + const chars2 = try diffLinesToCharsMunge2(allocator, text2, &line_array, &line_hash, UNICODE_ONE_THIRD); return .{ .chars_1 = chars1, .chars_2 = chars2, .line_array = line_array }; } @@ -1163,7 +1173,7 @@ fn diffIteratorToCharsMunge( max_segments: usize, ) DiffError![]const u8 { // This makes the unreachables in the function legitimate: - assert(max_segments <= 0x10ffff); // Maximum Unicode codepoint value. + assert(max_segments <= UNICODE_MAX); // Maximum Unicode codepoint value. var chars = ArrayListUnmanaged(u8){}; defer chars.deinit(allocator); var count: usize = 0; @@ -3727,21 +3737,40 @@ test "diffLinesToChars2" { try testing.expectEqualStrings("!", result.chars_2); // No linebreaks #2. try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // No linebreaks #3. result.deinit(allocator); -} - -test "workshop" { - const allocator = testing.allocator; - // Convert lines down to characters. - var tmp_array_list = std.ArrayList([]const u8).init(allocator); - defer tmp_array_list.deinit(); - try tmp_array_list.append("a"); - try tmp_array_list.append("b"); + { - var result = try diffLinesToChars2(allocator, "a", "b"); - try testing.expectEqualStrings(" ", result.chars_1); // No linebreaks #1. - try testing.expectEqualStrings("!", result.chars_2); // No linebreaks #2. - try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // No linebreaks #3. - result.deinit(allocator); + // TODO: More than 256 to reveal any 8-bit limitations but this requires + // some unicode logic that I don't want to deal with + // + // Casting to Unicode is straightforward and should sort correctly, I'm + // more concerned about the weird behavior when the 'char' is equal to a + // newline. Uncomment the EqualSlices below to see what I mean. + // I think there's some cleanup logic in the actual linediff that should + // take care of the problem, but I don't like it. + + const n: u21 = 1024; + + var line_list = std.ArrayList(u8).init(allocator); + defer line_list.deinit(); + var char_list = std.ArrayList(u8).init(allocator); + defer char_list.deinit(); + + var i: u21 = 32; + var char_buf: [4]u8 = undefined; + while (i < n) : (i += 1) { + const nbytes = std.unicode.wtf8Encode(i, &char_buf) catch unreachable; + try line_list.appendSlice(char_buf[0..nbytes]); + try line_list.append('\n'); + try char_list.appendSlice(char_buf[0..nbytes]); + } + const codepoint_len = std.unicode.utf8CountCodepoints(char_list.items) catch unreachable; + try testing.expectEqual(@as(usize, n - 32), codepoint_len); // Test initialization fail #2 + result = try diffLinesToChars2(allocator, line_list.items, ""); + defer result.deinit(allocator); + try testing.expectEqual(char_list.items.len, result.chars_1.len); + try testing.expectEqualSlices(u8, char_list.items, result.chars_1); + try testing.expectEqualStrings("", result.chars_2); + } } test diffLinesToChars { @@ -3769,7 +3798,6 @@ test diffLinesToChars { try testing.expectEqualStrings("", result.chars_1); // Empty string and blank lines #1 try testing.expectEqualStrings("\u{0001}\u{0002}\u{0003}\u{0003}", result.chars_2); // Empty string and blank lines #2 try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // Empty string and blank lines #3 - // ------ tmp_array_list.items.len = 0; try tmp_array_list.append(""); try tmp_array_list.append("a"); From 25d87ee003d4432ccdd5121aad87c05763994466 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 13:31:02 -0400 Subject: [PATCH 151/176] Comment justifying munge assertion --- DiffMatchPatch.zig | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index b5831e3..77a6f34 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1098,7 +1098,7 @@ fn diffLineMode2( // c0 control characters. The algorithm drops the bottom points, // not the top, that is, it will use 0x10ffff given enough unique // lines. -const UNICODE_MAX = 0x0010ffdf; +const UNICODE_MAX = 0x10ffdf; const UNICODE_TWO_THIRDS = 742724; const UNICODE_ONE_THIRD = 371355; comptime { @@ -1162,7 +1162,8 @@ fn diffLinesToCharsMunge2( /// @param iterator Returns the next segment. Must have functions /// next(), returning the next segment, and short_circuit(), /// called when max_segments is reached. -/// @param max_segments Maximum length of lineArray. +/// @param max_segments Maximum length of lineArray. Limited to +/// 0x10ffdf. /// @return Encoded string. fn diffIteratorToCharsMunge( allocator: std.mem.Allocator, @@ -1172,8 +1173,9 @@ fn diffIteratorToCharsMunge( iterator: anytype, max_segments: usize, ) DiffError![]const u8 { - // This makes the unreachables in the function legitimate: - assert(max_segments <= UNICODE_MAX); // Maximum Unicode codepoint value. + // Because we rebase the codepoint off the already counted segments, + // this makes the unreachables in the function legitimate: + assert(max_segments <= UNICODE_MAX); var chars = ArrayListUnmanaged(u8){}; defer chars.deinit(allocator); var count: usize = 0; From 9bab41e467760a29c99f75d7da005306aef79f89 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 13:39:42 -0400 Subject: [PATCH 152/176] Ports charsToLines This needs more extensive testing, but an integration suite is the best way to get that. --- DiffMatchPatch.zig | 55 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 77a6f34..f60d0f0 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1220,17 +1220,17 @@ fn diffCharsToLines2( for (diffs) |*d| { var cursor: usize = 0; while (cursor < d.text.len) { - const cp_len = std.unicode.utf8ByteSequenceLength(text[cursor]) catch { + const cp_len = std.unicode.utf8ByteSequenceLength(d.text[cursor]) catch { @panic("Internal decode error in diffsCharsToLines"); }; - const cp = try std.unicode.wtf8Decode(text[cursor..][0..cp_len]) catch { + const cp = std.unicode.wtf8Decode(d.text[cursor..][0..cp_len]) catch { @panic("Internal decode error in diffCharsToLines"); }; - try text.appendSlice(line_array[cp - 32]); + try text.appendSlice(allocator, line_array[cp - 32]); cursor += cp_len; } allocator.free(d.text); - d.text = try text.toOwnedSlice(); + d.text = try text.toOwnedSlice(allocator); } } @@ -3867,6 +3867,52 @@ fn testDiffCharsToLines( try testing.expectEqualDeep(params.expected, diffs.items); } +fn testDiffCharsToLines2( + allocator: std.mem.Allocator, + params: struct { + diffs: []const Diff, + line_array: []const []const u8, + expected: []const Diff, + }, +) !void { + var diffs = try DiffList.initCapacity(allocator, params.diffs.len); + defer deinitDiffList(allocator, &diffs); + + for (params.diffs) |item| { + diffs.appendAssumeCapacity(.{ .operation = item.operation, .text = try allocator.dupe(u8, item.text) }); + } + + try diffCharsToLines2(allocator, diffs.items, params.line_array); + + try testing.expectEqualDeep(params.expected, diffs.items); +} + +test "diffCharsToLines2" { + // Convert chars up to lines. + var diff_list = DiffList{}; + defer deinitDiffList(testing.allocator, &diff_list); + try diff_list.ensureTotalCapacity(testing.allocator, 2); + diff_list.appendSliceAssumeCapacity(&.{ + Diff.init(.equal, try testing.allocator.dupe(u8, " ! ")), + Diff.init(.insert, try testing.allocator.dupe(u8, "! !")), + }); + try testing.checkAllAllocationFailures( + testing.allocator, + testDiffCharsToLines2, + .{.{ + .diffs = diff_list.items, + .line_array = &[_][]const u8{ + "alpha\n", + "beta\n", + }, + .expected = &.{ + .{ .operation = .equal, .text = "alpha\nbeta\nalpha\n" }, + .{ .operation = .insert, .text = "beta\nalpha\nbeta\n" }, + }, + }}, + ); +} + test diffCharsToLines { // Convert chars up to lines. var diff_list = DiffList{}; @@ -3888,7 +3934,6 @@ test diffCharsToLines { .{ .operation = .insert, .text = "beta\nalpha\nbeta\n" }, }, }}); - // TODO: Implement exhaustive tests } From 6e721cb2cb8803a32ece04121226929eab8cb48d Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 13:49:17 -0400 Subject: [PATCH 153/176] Delete old version, rename new ones --- DiffMatchPatch.zig | 346 ++++----------------------------------------- 1 file changed, 24 insertions(+), 322 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index f60d0f0..00b1aec 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -900,200 +900,6 @@ fn diffLineMode( return diffs; } -const LinesToCharsResult = struct { - chars_1: []const u8, - chars_2: []const u8, - line_array: ArrayListUnmanaged([]const u8), - - pub fn deinit(self: *LinesToCharsResult, allocator: Allocator) void { - allocator.free(self.chars_1); - allocator.free(self.chars_2); - self.line_array.deinit(allocator); - } -}; - -/// Split two texts into a list of strings. Reduce the texts to a string of -/// hashes where each Unicode character represents one line. -/// @param text1 First string. -/// @param text2 Second string. -/// @return Three element Object array, containing the encoded text1, the -/// encoded text2 and the List of unique strings. The zeroth element -/// of the List of unique strings is intentionally blank. -fn diffLinesToChars( - allocator: std.mem.Allocator, - text1: []const u8, - text2: []const u8, -) DiffError!LinesToCharsResult { - var line_array = ArrayListUnmanaged([]const u8){}; - errdefer line_array.deinit(allocator); - var line_hash = std.StringHashMapUnmanaged(usize){}; - defer line_hash.deinit(allocator); - // e.g. line_array[4] == "Hello\n" - // e.g. line_hash.get("Hello\n") == 4 - - // "\x00" is a valid character, but various debuggers don't like it. - // So we'll insert a junk entry to avoid generating a null character. - try line_array.append(allocator, ""); - - // Allocate 2/3rds of the space for text1, the rest for text2. - const chars1 = try diffLinesToCharsMunge(allocator, text1, &line_array, &line_hash, 170); - const chars2 = try diffLinesToCharsMunge(allocator, text2, &line_array, &line_hash, 255); - return .{ .chars_1 = chars1, .chars_2 = chars2, .line_array = line_array }; -} - -/// Split a text into a list of strings. Reduce the texts to a string of -/// hashes where each Unicode character represents one line. -/// @param text String to encode. -/// @param lineArray List of unique strings. -/// @param lineHash Map of strings to indices. -/// @param maxLines Maximum length of lineArray. -/// @return Encoded string. -fn diffLinesToCharsMunge( - allocator: std.mem.Allocator, - text: []const u8, - line_array: *ArrayListUnmanaged([]const u8), - line_hash: *std.StringHashMapUnmanaged(usize), - max_lines: usize, -) DiffError![]const u8 { - var line_start: isize = 0; - var line_end: isize = -1; - var line: []const u8 = undefined; - var chars = ArrayListUnmanaged(u8){}; - defer chars.deinit(allocator); - // Walk the text, pulling out a Substring for each line. - while (line_end < @as(isize, @intCast(text.len)) - 1) { - line_end = b: { - break :b @as(isize, @intCast(std.mem.indexOf(u8, text[@intCast(line_start)..], "\n") orelse - break :b @intCast(text.len - 1))) + line_start; - }; - line = text[@intCast(line_start) .. @as(usize, @intCast(line_start)) + @as(usize, @intCast(line_end + 1 - line_start))]; - - if (line_hash.get(line)) |value| { - try chars.append(allocator, @intCast(value)); - } else { - if (line_array.items.len == max_lines) { - // Bail out at 255 because char 256 == char 0. - line = text[@intCast(line_start)..]; - line_end = @intCast(text.len); - } - try line_array.append(allocator, line); - try line_hash.put(allocator, line, line_array.items.len - 1); - try chars.append(allocator, @intCast(line_array.items.len - 1)); - } - line_start = line_end + 1; - } - return try chars.toOwnedSlice(allocator); -} - -/// Rehydrate the text in a diff from a string of line hashes to real lines -/// of text. -/// @param diffs List of Diff objects. -/// @param lineArray List of unique strings. -fn diffCharsToLines( - allocator: std.mem.Allocator, - diffs: []Diff, - line_array: []const []const u8, -) DiffError!void { - var text = ArrayListUnmanaged(u8){}; - defer text.deinit(allocator); - - for (diffs) |*d| { - var j: usize = 0; - while (j < d.text.len) : (j += 1) { - try text.appendSlice(allocator, line_array[d.text[j]]); - } - allocator.free(d.text); - d.text = try text.toOwnedSlice(allocator); - } -} - -/// Do a quick line-level diff on both strings, then rediff the parts for -/// greater accuracy. -/// This speedup can produce non-minimal diffs. -/// @param text1 Old string to be diffed. -/// @param text2 New string to be diffed. -/// @param deadline Time when the diff should be complete by. -/// @return List of Diff objects. -fn diffLineMode2( - dmp: DiffMatchPatch, - allocator: std.mem.Allocator, - text1_in: []const u8, - text2_in: []const u8, - deadline: u64, -) DiffError!DiffList { - // Scan the text on a line-by-line basis first. - var a = try diffLinesToChars2(allocator, text1_in, text2_in); - defer a.deinit(allocator); - const text1 = a.chars_1; - const text2 = a.chars_2; - const line_array = a.line_array; - - var diffs: DiffList = try dmp.diffInternal(allocator, text1, text2, false, deadline); - errdefer diffs.deinit(allocator); - // Convert the diff back to original text. - try diffCharsToLines2(allocator, diffs.items, line_array.items); - // Eliminate freak matches (e.g. blank lines) - try diffCleanupSemantic(allocator, &diffs); - - // Rediff any replacement blocks, this time character-by-character. - // Add a dummy entry at the end. - try diffs.append(allocator, Diff.init(.equal, "")); - - var pointer: usize = 0; - var count_delete: usize = 0; - var count_insert: usize = 0; - var text_delete = ArrayListUnmanaged(u8){}; - var text_insert = ArrayListUnmanaged(u8){}; - defer { - text_delete.deinit(allocator); - text_insert.deinit(allocator); - } - - while (pointer < diffs.items.len) { - switch (diffs.items[pointer].operation) { - .insert => { - count_insert += 1; - try text_insert.appendSlice(allocator, diffs.items[pointer].text); - }, - .delete => { - count_delete += 1; - try text_delete.appendSlice(allocator, diffs.items[pointer].text); - }, - .equal => { - // Upon reaching an equality, check for prior redundancies. - if (count_delete >= 1 and count_insert >= 1) { - // Delete the offending records and add the merged ones. - freeRangeDiffList( - allocator, - &diffs, - pointer - count_delete - count_insert, - count_delete + count_insert, - ); - try diffs.replaceRange( - allocator, - pointer - count_delete - count_insert, - count_delete + count_insert, - &.{}, - ); - pointer = pointer - count_delete - count_insert; - var sub_diff = try dmp.diffInternal(allocator, text_delete.items, text_insert.items, false, deadline); - defer sub_diff.deinit(allocator); - try diffs.insertSlice(allocator, pointer, sub_diff.items); - pointer = pointer + sub_diff.items.len; - } - count_insert = 0; - count_delete = 0; - text_delete.items.len = 0; - text_insert.items.len = 0; - }, - } - pointer += 1; - } - diffs.items.len -= 1; // Remove the dummy entry at the end. - - return diffs; -} - // These numbers have a 32 point buffer, to avoid annoyance with // c0 control characters. The algorithm drops the bottom points, // not the top, that is, it will use 0x10ffff given enough unique @@ -1111,7 +917,7 @@ comptime { /// @return Three element Object array, containing the encoded text1, the /// encoded text2 and the List of unique strings. The zeroth element /// of the List of unique strings is intentionally blank. -fn diffLinesToChars2( +fn diffLinesToChars( allocator: std.mem.Allocator, text1: []const u8, text2: []const u8, @@ -1124,11 +930,23 @@ fn diffLinesToChars2( // e.g. line_hash.get("Hello\n") == 4 // Allocate 2/3rds of the space for text1, the rest for text2. - const chars1 = try diffLinesToCharsMunge2(allocator, text1, &line_array, &line_hash, UNICODE_TWO_THIRDS); - const chars2 = try diffLinesToCharsMunge2(allocator, text2, &line_array, &line_hash, UNICODE_ONE_THIRD); + const chars1 = try diffLinesToCharsMunge(allocator, text1, &line_array, &line_hash, UNICODE_TWO_THIRDS); + const chars2 = try diffLinesToCharsMunge(allocator, text2, &line_array, &line_hash, UNICODE_ONE_THIRD); return .{ .chars_1 = chars1, .chars_2 = chars2, .line_array = line_array }; } +const LinesToCharsResult = struct { + chars_1: []const u8, + chars_2: []const u8, + line_array: ArrayListUnmanaged([]const u8), + + pub fn deinit(self: *LinesToCharsResult, allocator: Allocator) void { + allocator.free(self.chars_1); + allocator.free(self.chars_2); + self.line_array.deinit(allocator); + } +}; + /// Split a text into a list of strings. Reduce the texts to a string of /// hashes where each Unicode character represents one line. /// @param text String to encode. @@ -1136,7 +954,7 @@ fn diffLinesToChars2( /// @param lineHash Map of strings to indices. /// @param maxLines Maximum length of lineArray. /// @return Encoded string. -fn diffLinesToCharsMunge2( +fn diffLinesToCharsMunge( allocator: std.mem.Allocator, text: []const u8, line_array: *ArrayListUnmanaged([]const u8), @@ -1210,7 +1028,7 @@ fn diffIteratorToCharsMunge( /// of text. /// @param diffs List of Diff objects. /// @param lineArray List of unique strings. -fn diffCharsToLines2( +fn diffCharsToLines( allocator: Allocator, diffs: []Diff, line_array: []const []const u8, @@ -3705,7 +3523,7 @@ test diffHalfMatch { }}); } -test "diffLinesToChars2" { +test diffLinesToChars { const allocator = testing.allocator; // Convert lines down to characters. var tmp_array_list = std.ArrayList([]const u8).init(allocator); @@ -3713,7 +3531,7 @@ test "diffLinesToChars2" { try tmp_array_list.append("alpha\n"); try tmp_array_list.append("beta\n"); - var result = try diffLinesToChars2(allocator, "alpha\nbeta\nalpha\n", "beta\nalpha\nbeta\n"); + var result = try diffLinesToChars(allocator, "alpha\nbeta\nalpha\n", "beta\nalpha\nbeta\n"); try testing.expectEqualStrings(" ! ", result.chars_1); // Shared lines #1 try testing.expectEqualStrings("! !", result.chars_2); // Shared lines #2 @@ -3725,7 +3543,7 @@ test "diffLinesToChars2" { try tmp_array_list.append("beta\r\n"); try tmp_array_list.append("\r\n"); - result = try diffLinesToChars2(allocator, "", "alpha\r\nbeta\r\n\r\n\r\n"); + result = try diffLinesToChars(allocator, "", "alpha\r\nbeta\r\n\r\n\r\n"); try testing.expectEqualStrings("", result.chars_1); // Empty string and blank lines #1 try testing.expectEqualStrings(" !\"\"", result.chars_2); // Empty string and blank lines #2 try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // Empty string and blank lines #3 @@ -3734,7 +3552,7 @@ test "diffLinesToChars2" { try tmp_array_list.append("a"); try tmp_array_list.append("b"); - result = try diffLinesToChars2(allocator, "a", "b"); + result = try diffLinesToChars(allocator, "a", "b"); try testing.expectEqualStrings(" ", result.chars_1); // No linebreaks #1. try testing.expectEqualStrings("!", result.chars_2); // No linebreaks #2. try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // No linebreaks #3. @@ -3767,7 +3585,7 @@ test "diffLinesToChars2" { } const codepoint_len = std.unicode.utf8CountCodepoints(char_list.items) catch unreachable; try testing.expectEqual(@as(usize, n - 32), codepoint_len); // Test initialization fail #2 - result = try diffLinesToChars2(allocator, line_list.items, ""); + result = try diffLinesToChars(allocator, line_list.items, ""); defer result.deinit(allocator); try testing.expectEqual(char_list.items.len, result.chars_1.len); try testing.expectEqualSlices(u8, char_list.items, result.chars_1); @@ -3775,78 +3593,6 @@ test "diffLinesToChars2" { } } -test diffLinesToChars { - const allocator = testing.allocator; - // Convert lines down to characters. - var tmp_array_list = std.ArrayList([]const u8).init(allocator); - defer tmp_array_list.deinit(); - try tmp_array_list.append(""); - try tmp_array_list.append("alpha\n"); - try tmp_array_list.append("beta\n"); - - var result = try diffLinesToChars(allocator, "alpha\nbeta\nalpha\n", "beta\nalpha\nbeta\n"); - try testing.expectEqualStrings("\u{0001}\u{0002}\u{0001}", result.chars_1); // Shared lines #1 - try testing.expectEqualStrings("\u{0002}\u{0001}\u{0002}", result.chars_2); // Shared lines #2 - try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // Shared lines #3 - - tmp_array_list.items.len = 0; - try tmp_array_list.append(""); - try tmp_array_list.append("alpha\r\n"); - try tmp_array_list.append("beta\r\n"); - try tmp_array_list.append("\r\n"); - result.deinit(allocator); - - result = try diffLinesToChars(allocator, "", "alpha\r\nbeta\r\n\r\n\r\n"); - try testing.expectEqualStrings("", result.chars_1); // Empty string and blank lines #1 - try testing.expectEqualStrings("\u{0001}\u{0002}\u{0003}\u{0003}", result.chars_2); // Empty string and blank lines #2 - try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // Empty string and blank lines #3 - tmp_array_list.items.len = 0; - try tmp_array_list.append(""); - try tmp_array_list.append("a"); - try tmp_array_list.append("b"); - result.deinit(allocator); - - result = try diffLinesToChars(allocator, "a", "b"); - try testing.expectEqualStrings("\u{0001}", result.chars_1); // No linebreaks #1. - try testing.expectEqualStrings("\u{0002}", result.chars_2); // No linebreaks #2. - try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // No linebreaks #3. - result.deinit(allocator); - - // TODO: More than 256 to reveal any 8-bit limitations but this requires - // some unicode logic that I don't want to deal with - // - // Casting to Unicode is straightforward and should sort correctly, I'm - // more concerned about the weird behavior when the 'char' is equal to a - // newline. Uncomment the EqualSlices below to see what I mean. - // I think there's some cleanup logic in the actual linediff that should - // take care of the problem, but I don't like it. - - const n: u8 = 255; - tmp_array_list.items.len = 0; - - var line_list = std.ArrayList(u8).init(allocator); - defer line_list.deinit(); - var char_list = std.ArrayList(u8).init(allocator); - defer char_list.deinit(); - - var i: u8 = 1; - while (i < n) : (i += 1) { - try tmp_array_list.append(&.{ i, '\n' }); - try line_list.appendSlice(&.{ i, '\n' }); - try char_list.append(i); - } - try testing.expectEqual(@as(usize, n - 1), tmp_array_list.items.len); // Test initialization fail #1 - try testing.expectEqual(@as(usize, n - 1), char_list.items.len); // Test initialization fail #2 - try tmp_array_list.insert(0, ""); - result = try diffLinesToChars(allocator, line_list.items, ""); - defer result.deinit(allocator); - // TODO: This isn't equal, should it be? - // try testing.expectEqualSlices(u8, char_list.items, result.chars_1); - try testing.expectEqualStrings("", result.chars_2); - // TODO this is wrong because of the max_value I think? - // try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); -} - fn testDiffCharsToLines( allocator: std.mem.Allocator, params: struct { @@ -3867,27 +3613,7 @@ fn testDiffCharsToLines( try testing.expectEqualDeep(params.expected, diffs.items); } -fn testDiffCharsToLines2( - allocator: std.mem.Allocator, - params: struct { - diffs: []const Diff, - line_array: []const []const u8, - expected: []const Diff, - }, -) !void { - var diffs = try DiffList.initCapacity(allocator, params.diffs.len); - defer deinitDiffList(allocator, &diffs); - - for (params.diffs) |item| { - diffs.appendAssumeCapacity(.{ .operation = item.operation, .text = try allocator.dupe(u8, item.text) }); - } - - try diffCharsToLines2(allocator, diffs.items, params.line_array); - - try testing.expectEqualDeep(params.expected, diffs.items); -} - -test "diffCharsToLines2" { +test diffCharsToLines { // Convert chars up to lines. var diff_list = DiffList{}; defer deinitDiffList(testing.allocator, &diff_list); @@ -3898,7 +3624,7 @@ test "diffCharsToLines2" { }); try testing.checkAllAllocationFailures( testing.allocator, - testDiffCharsToLines2, + testDiffCharsToLines, .{.{ .diffs = diff_list.items, .line_array = &[_][]const u8{ @@ -3913,30 +3639,6 @@ test "diffCharsToLines2" { ); } -test diffCharsToLines { - // Convert chars up to lines. - var diff_list = DiffList{}; - defer deinitDiffList(testing.allocator, &diff_list); - try diff_list.ensureTotalCapacity(testing.allocator, 2); - diff_list.appendSliceAssumeCapacity(&.{ - Diff.init(.equal, try testing.allocator.dupe(u8, "\u{0001}\u{0002}\u{0001}")), - Diff.init(.insert, try testing.allocator.dupe(u8, "\u{0002}\u{0001}\u{0002}")), - }); - try testing.checkAllAllocationFailures(testing.allocator, testDiffCharsToLines, .{.{ - .diffs = diff_list.items, - .line_array = &[_][]const u8{ - "", - "alpha\n", - "beta\n", - }, - .expected = &.{ - .{ .operation = .equal, .text = "alpha\nbeta\nalpha\n" }, - .{ .operation = .insert, .text = "beta\nalpha\nbeta\n" }, - }, - }}); - // TODO: Implement exhaustive tests -} - fn testDiffCleanupMerge(allocator: std.mem.Allocator, params: struct { input: []const Diff, expected: []const Diff, From 6b237e487c1725b89a69ea520525f6fb51f5e0be Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 13:55:08 -0400 Subject: [PATCH 154/176] Replace 'magic number' 32 with a constant --- DiffMatchPatch.zig | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 00b1aec..5793e3c 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -907,8 +907,10 @@ fn diffLineMode( const UNICODE_MAX = 0x10ffdf; const UNICODE_TWO_THIRDS = 742724; const UNICODE_ONE_THIRD = 371355; +const CHAR_OFFSET = 32; comptime { assert(UNICODE_TWO_THIRDS + UNICODE_ONE_THIRD == UNICODE_MAX); + assert(UNICODE_TWO_THIRDS + UNICODE_ONE_THIRD + CHAR_OFFSET == 0x10ffff); } /// Split two texts into a list of strings. Reduce the texts to a string of /// hashes where each Unicode character represents one line. @@ -997,7 +999,7 @@ fn diffIteratorToCharsMunge( var chars = ArrayListUnmanaged(u8){}; defer chars.deinit(allocator); var count: usize = 0; - var codepoint: u21 = 32 + cast(u21, segment_array.items.len); + var codepoint: u21 = CHAR_OFFSET + cast(u21, segment_array.items.len); var char_buf: [4]u8 = undefined; while (iterator.next()) |line| { if (segment_hash.get(line)) |value| { @@ -1005,7 +1007,7 @@ fn diffIteratorToCharsMunge( try chars.appendSlice(allocator, char_buf[0..nbytes]); count += line.len; } else { - if (codepoint - 32 == max_segments) { + if (codepoint - CHAR_OFFSET == max_segments) { // Bail out iterator.short_circuit(); const final_line = text[count..]; @@ -1044,7 +1046,7 @@ fn diffCharsToLines( const cp = std.unicode.wtf8Decode(d.text[cursor..][0..cp_len]) catch { @panic("Internal decode error in diffCharsToLines"); }; - try text.appendSlice(allocator, line_array[cp - 32]); + try text.appendSlice(allocator, line_array[cp - CHAR_OFFSET]); cursor += cp_len; } allocator.free(d.text); @@ -3575,7 +3577,7 @@ test diffLinesToChars { var char_list = std.ArrayList(u8).init(allocator); defer char_list.deinit(); - var i: u21 = 32; + var i: u21 = CHAR_OFFSET; var char_buf: [4]u8 = undefined; while (i < n) : (i += 1) { const nbytes = std.unicode.wtf8Encode(i, &char_buf) catch unreachable; @@ -3584,7 +3586,7 @@ test diffLinesToChars { try char_list.appendSlice(char_buf[0..nbytes]); } const codepoint_len = std.unicode.utf8CountCodepoints(char_list.items) catch unreachable; - try testing.expectEqual(@as(usize, n - 32), codepoint_len); // Test initialization fail #2 + try testing.expectEqual(@as(usize, n - CHAR_OFFSET), codepoint_len); // Test initialization fail #2 result = try diffLinesToChars(allocator, line_list.items, ""); defer result.deinit(allocator); try testing.expectEqual(char_list.items.len, result.chars_1.len); From 09c7e01c3715ddd2818396df720cf342314753e9 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 14:05:03 -0400 Subject: [PATCH 155/176] Updates roadmap --- roadmap.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/roadmap.md b/roadmap.md index 7d59140..e33a91f 100644 --- a/roadmap.md +++ b/roadmap.md @@ -4,11 +4,11 @@ - [✅] Add DiffMatchPatch object instead of @This() (which won't work) - [✅] Port match. - [✅] Port test coverage. -- [ ] Unicode-aware `diffLineMode`. - - [ ] Coverage for all corner cases of preventing diff splits which aren't +- [✅] Unicode-aware `diffLineMode`. + - [✅] Coverage for all corner cases of preventing diff splits which aren't on valid UTF-8 boundaries. - - [ ] Convert `line_array` to encode UTF-8 byte sequences and store `u21` keys - - [ ] Make the inner function accept a stream iterator, one which delivers the + - [✅] Convert `line_array` to encode UTF-8 byte sequences and store `u21` keys + - [✅] Make the inner function accept a stream iterator, one which delivers the entire string with boundaries (where applicable) at the end. - [ ] Refactor: the port currently treats Diffs and Patches as raw ArrayLists, these should be proper Zig objects, with member functions, and probably @@ -34,6 +34,9 @@ - [ ] Histogram? - [ ] Imara diff has an optimized histogram: https://github.com/pascalkuthe/imara-diff + - [ ] Calculating the histogram while hashing the lines would be + straightforward, this could be comptime-gated, but probably + just a second copy of the munge function is fine. - [ ] POSIX-diff compatible patch output? - [ ] This one seems pretty worthwhile to me. It would need to call line mode without refining further, but everything else is fairly simple. From 9191e63b63d52ce7203e7b2c518a41a0d9da30f7 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 14:05:16 -0400 Subject: [PATCH 156/176] Catch infinite loop bug in decodeUri Coverage to the rescue once again: the tests were calling this function without ever decoding anything. --- DiffMatchPatch.zig | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 5793e3c..2d584a8 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1722,6 +1722,7 @@ fn diffCommonOverlap(text1_in: []const u8, text2_in: []const u8) usize { // should not be possible. // I'm going to add a panic just so I know if test cases of any sort // trigger this code path. + // XXX Remove this before merge if it can't be triggered. if (text2[best_idx] >= 0x80 and is_follow(text2[best_idx + 1])) { if (true) { @panic("Your assumption regarding diffCommonOverlap is invalid!"); @@ -3225,7 +3226,7 @@ fn decodeUri(allocator: Allocator, line: []const u8) ![]const u8 { ) catch return error.BadPatchString; try new_line.append(codeunit[0]); var cursor = first + 3; - while (std.mem.indexOf(u8, line[cursor..], "%")) |next| { + while (std.mem.indexOfScalarPos(u8, line, cursor, '%')) |next| { codeunit = std.fmt.hexToBytes( &out_buf, line[next + 1 .. next + 3], @@ -3314,7 +3315,7 @@ fn encodeUri(allocator: std.mem.Allocator, text: []const u8) ![]u8 { //| TESTS //| -test encodeUri { +test "encodeUri" { const allocator = std.testing.allocator; const special_chars = "!#$&'()*+,-./:;=?@_~"; const special_encoded = try encodeUri(allocator, special_chars); @@ -3325,9 +3326,12 @@ test encodeUri { defer allocator.free(alpha_encoded); try testing.expectEqualStrings(alphaspace, alpha_encoded); const to_encode = "\"%<>[\\]^`{|}δ"; - const encodes = try encodeUri(allocator, to_encode); - defer allocator.free(encodes); - try testing.expectEqualStrings("%22%25%3C%3E%5B%5C%5D%5E%60%7B%7C%7D%CE%B4", encodes); + const encoded = try encodeUri(allocator, to_encode); + defer allocator.free(encoded); + try testing.expectEqualStrings("%22%25%3C%3E%5B%5C%5D%5E%60%7B%7C%7D%CE%B4", encoded); + const decoded = try decodeUri(allocator, encoded); + defer allocator.free(decoded); + try testing.expectEqualStrings(to_encode, decoded); } test diffCommonPrefix { From 39012aa73e17b26c5543d80fda73f0983e3c092d Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 15:57:38 -0400 Subject: [PATCH 157/176] Line coverage of early stoppage This also refactors the iterator to get rid of the separate text and count, and uses the short-circuit algorithm to get the rest of the text, making it the source of truth. --- DiffMatchPatch.zig | 62 ++++++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 2d584a8..9be344d 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -926,6 +926,7 @@ fn diffLinesToChars( ) DiffError!LinesToCharsResult { var line_array = ArrayListUnmanaged([]const u8){}; errdefer line_array.deinit(allocator); + line_array.items.len = 0; var line_hash = std.StringHashMapUnmanaged(u21){}; defer line_hash.deinit(allocator); // e.g. line_array[4] == "Hello\n" @@ -966,7 +967,6 @@ fn diffLinesToCharsMunge( var iter = LineIterator{ .text = text }; return try diffIteratorToCharsMunge( allocator, - text, line_array, line_hash, &iter, @@ -974,9 +974,17 @@ fn diffLinesToCharsMunge( ); } -/// Split a text into segments. Reduce the texts to a string of -/// hashes where each Unicode character represents one segment. -/// @param text String to encode. +/// Split a text into segments, yielded from an iterator. +/// Reduce the texts to a string of hashes where each Unicode character +/// represents one segment. +/// +/// Iterators must provide: `next()`, which gives the next segment of +/// the test, and `short_circuit(usize)`, which is called when the +/// segment limit is reached, and returns the rest of the text. The +/// parameter provided will be the length of the last segment provided +/// by `next()`, since the function will not process that segment, and +/// its text must be included in the remainder. +/// /// @param segment_array List of unique string segments. /// @param line_hash Map of strings to indices into segment_array. /// @param iterator Returns the next segment. Must have functions @@ -987,7 +995,6 @@ fn diffLinesToCharsMunge( /// @return Encoded string. fn diffIteratorToCharsMunge( allocator: std.mem.Allocator, - text: []const u8, segment_array: *ArrayListUnmanaged([]const u8), segment_hash: *std.StringHashMapUnmanaged(u21), iterator: anytype, @@ -998,23 +1005,21 @@ fn diffIteratorToCharsMunge( assert(max_segments <= UNICODE_MAX); var chars = ArrayListUnmanaged(u8){}; defer chars.deinit(allocator); - var count: usize = 0; var codepoint: u21 = CHAR_OFFSET + cast(u21, segment_array.items.len); var char_buf: [4]u8 = undefined; while (iterator.next()) |line| { if (segment_hash.get(line)) |value| { const nbytes = std.unicode.wtf8Encode(value, &char_buf) catch unreachable; try chars.appendSlice(allocator, char_buf[0..nbytes]); - count += line.len; } else { if (codepoint - CHAR_OFFSET == max_segments) { // Bail out - iterator.short_circuit(); - const final_line = text[count..]; + const final_line = iterator.short_circuit(line.len); try segment_array.append(allocator, final_line); try segment_hash.put(allocator, final_line, codepoint); const nbytes = std.unicode.wtf8Encode(codepoint, &char_buf) catch unreachable; try chars.appendSlice(allocator, char_buf[0..nbytes]); + break; } try segment_array.append(allocator, line); try segment_hash.put(allocator, line, codepoint); @@ -1078,8 +1083,12 @@ const LineIterator = struct { } } - pub fn short_circuit(iter: *LineIterator) void { + /// Terminate the iterator early by returning all remaining text. + /// `back_out` parameter is how far before the cursor to slice from. + pub fn short_circuit(iter: *LineIterator, back_out: usize) []const u8 { + const from = iter.cursor - back_out; iter.cursor = iter.text.len; + return iter.text[from..]; } }; @@ -3538,7 +3547,6 @@ test diffLinesToChars { try tmp_array_list.append("beta\n"); var result = try diffLinesToChars(allocator, "alpha\nbeta\nalpha\n", "beta\nalpha\nbeta\n"); - try testing.expectEqualStrings(" ! ", result.chars_1); // Shared lines #1 try testing.expectEqualStrings("! !", result.chars_2); // Shared lines #2 try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // Shared lines #3 @@ -3563,17 +3571,8 @@ test diffLinesToChars { try testing.expectEqualStrings("!", result.chars_2); // No linebreaks #2. try testing.expectEqualDeep(tmp_array_list.items, result.line_array.items); // No linebreaks #3. result.deinit(allocator); - { - - // TODO: More than 256 to reveal any 8-bit limitations but this requires - // some unicode logic that I don't want to deal with - // - // Casting to Unicode is straightforward and should sort correctly, I'm - // more concerned about the weird behavior when the 'char' is equal to a - // newline. Uncomment the EqualSlices below to see what I mean. - // I think there's some cleanup logic in the actual linediff that should - // take care of the problem, but I don't like it. + { const n: u21 = 1024; var line_list = std.ArrayList(u8).init(allocator); @@ -3590,12 +3589,27 @@ test diffLinesToChars { try char_list.appendSlice(char_buf[0..nbytes]); } const codepoint_len = std.unicode.utf8CountCodepoints(char_list.items) catch unreachable; - try testing.expectEqual(@as(usize, n - CHAR_OFFSET), codepoint_len); // Test initialization fail #2 + try testing.expectEqual(@as(usize, n - CHAR_OFFSET), codepoint_len); result = try diffLinesToChars(allocator, line_list.items, ""); - defer result.deinit(allocator); try testing.expectEqual(char_list.items.len, result.chars_1.len); try testing.expectEqualSlices(u8, char_list.items, result.chars_1); try testing.expectEqualStrings("", result.chars_2); + result.deinit(allocator); + + // Test iterator stop + // TODO this isn't a complete test, it verifies that iteration + // stops, but not that it does so correctly. + var line_array = ArrayListUnmanaged([]const u8){}; + defer line_array.deinit(allocator); + line_array.items.len = 0; + var line_hash = std.StringHashMapUnmanaged(u21){}; + defer line_hash.deinit(allocator); + const char_out = try diffLinesToCharsMunge(allocator, line_list.items, &line_array, &line_hash, 950); + defer allocator.free(char_out); + try testing.expectEqualStrings( + "ϖ\nϗ\nϘ\nϙ\nϚ\nϛ\nϜ\nϝ\nϞ\nϟ\nϠ\nϡ\nϢ\nϣ\nϤ\nϥ\nϦ\nϧ\nϨ\nϩ\nϪ\nϫ\nϬ\nϭ\nϮ\nϯ\nϰ\nϱ\nϲ\nϳ\nϴ\nϵ\n϶\nϷ\nϸ\nϹ\nϺ\nϻ\nϼ\nϽ\nϾ\nϿ\n", + line_array.getLast(), + ); } } @@ -4882,7 +4896,7 @@ fn testDiffCleanupEfficiency( try testing.expectEqualDeep(params.expected, diffs.items); } -test "diffCleanupEfficiency" { +test diffCleanupEfficiency { const allocator = testing.allocator; var dmp = DiffMatchPatch{}; dmp.diff_edit_cost = 4; From 6787aa6310a67559417a9e8a2d41fffd7c55e513 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 22:35:23 -0400 Subject: [PATCH 158/176] Remove unused Bitap 2.0 functions Also removes the Python script for building integration tests. These have all been moved to their own branch. --- DiffMatchPatch.zig | 178 +++------------------------------------------ make-file-list.py | 26 ------- 2 files changed, 11 insertions(+), 193 deletions(-) delete mode 100644 make-file-list.py diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 9be344d..164a0b6 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -882,7 +882,13 @@ fn diffLineMode( &.{}, ); pointer = pointer - count_delete - count_insert; - var sub_diff = try dmp.diffInternal(allocator, text_delete.items, text_insert.items, false, deadline); + var sub_diff = try dmp.diffInternal( + allocator, + text_delete.items, + text_insert.items, + false, + deadline, + ); defer sub_diff.deinit(allocator); try diffs.insertSlice(allocator, pointer, sub_diff.items); pointer = pointer + sub_diff.items.len; @@ -1064,6 +1070,7 @@ const LineIterator = struct { cursor: usize = 0, text: []const u8, + /// Return the next line, including its newline, if one is present. pub fn next(iter: *LineIterator) ?[]const u8 { if (iter.cursor == iter.text.len) return null; const maybe_newline = std.mem.indexOfScalarPos( @@ -1436,13 +1443,11 @@ pub fn diffCleanupSemanticLossless( // First, shift the edit as far left as possible. const common_offset = diffCommonSuffix(equality_1.items, edit.items); if (common_offset > 0) { - // TODO: Use buffer const common_string = try allocator.dupe(u8, edit.items[edit.items.len - common_offset ..]); defer allocator.free(common_string); equality_1.items.len = equality_1.items.len - common_offset; - // edit.items.len = edit.items.len - common_offset; const not_common = try allocator.dupe(u8, edit.items[0 .. edit.items.len - common_offset]); defer allocator.free(not_common); @@ -1551,10 +1556,8 @@ fn diffCleanupSemanticScore(one: []const u8, two: []const u8) usize { const lineBreak1 = whitespace1 and std.ascii.isControl(char1); const lineBreak2 = whitespace2 and std.ascii.isControl(char2); const blankLine1 = lineBreak1 and - // BLANKLINEEND.IsMatch(one); (std.mem.endsWith(u8, one, "\n\n") or std.mem.endsWith(u8, one, "\n\r\n")); const blankLine2 = lineBreak2 and - // BLANKLINESTART.IsMatch(two); (std.mem.startsWith(u8, two, "\n\n") or std.mem.startsWith(u8, two, "\r\n\n") or std.mem.startsWith(u8, two, "\n\r\n") or @@ -1584,7 +1587,7 @@ inline fn boolInt(b: bool) u8 { } /// Reduce the number of edits by eliminating operationally trivial -/// equalities. TODO this needs tests +/// equalities. pub fn diffCleanupEfficiency( dmp: DiffMatchPatch, allocator: std.mem.Allocator, @@ -1732,31 +1735,12 @@ fn diffCommonOverlap(text1_in: []const u8, text2_in: []const u8) usize { // I'm going to add a panic just so I know if test cases of any sort // trigger this code path. // XXX Remove this before merge if it can't be triggered. - if (text2[best_idx] >= 0x80 and is_follow(text2[best_idx + 1])) { + if (is_follow(text2[best_idx])) { if (true) { @panic("Your assumption regarding diffCommonOverlap is invalid!"); } // back out - assert(best_idx == best); - if (!is_follow(text2[best])) { - // It's a lead, one back is fine - return best - 1; - } - best -= 1; - if (best == 0) return 0; - // It's ok to get no overlap, so we ignore malformation: - // a bunch of follows could walk back to zero, and that's - // fine with us - while (is_follow(text2[best])) { - best -= 1; - if (best == 0) return 0; - } - // should be a lead, but ASCII is fine, so - if (text2[best] < 0x80) { - return best; - } else { - return best - 1; - } + return fixSplitBackward(text2, best_idx); } return best_idx; } @@ -2018,125 +2002,6 @@ pub fn matchMain( } } -// TODO doubling the bits to fit in usize is nice and all, but there's no -// reason to be limited to that, we have bitsets which can be as large as -// we'd like. This could be passed a comptime power-of-two size, and use -// that to make an ArrayBitSet specialized for several sizes, up to, IDK, -// 2k? Then split very large patches only. 64, 256, 512, 1024, 2028, is -// a nice balance between code size and versatility. -// Something like this: -fn matchBitapImproved( - dmp: DiffMatchPatch, - allocator: Allocator, - text: []const u8, - pattern: []const u8, - loc: usize, - UIntType: type, -) ?usize { - assert(pattern.len < @bitSizeOf(UIntType)); - const ShiftWidth = ShiftSizeForType(UIntType); - // Initialise the alphabet. - var map = try matchAlphabet(allocator, pattern); - defer map.deinit(); - // Highest score beyond which we give up. - var threshold = dmp.threshold; - // Is there a nearby exact match? (speedup) - var best_loc = std.mem.indexOfPos(u8, text, pattern); - if (best_loc) |best| { - threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), threshold); - } - // What about in the other direction? (speedup) - const trunc_text = text[0..@min(loc + pattern.len, text.len)]; - best_loc = std.mem.lastIndexOf(u8, trunc_text, pattern); - if (best_loc) |best| { - threshold = @min(dmp.matchBitapScore(0, best, loc, pattern), threshold); - } - // Initialise the bit arrays. - const shift: ShiftWidth = @intCast(pattern.len - 1); - // 0 for a match for faster bit twiddles - const matchmask = ~(1 << shift); - best_loc = null; - var bin_min: usize = undefined; - var bin_mid: usize = undefined; - var bin_max = pattern.len + text.len; - // null last_rd to simplying freeing memory - var last_rd = try allocator.alloc(UIntType, 0); - for (0..pattern.len) |d| { - // Scan for the best match; each iteration allows for one more error. - // Run a binary search to determine how far from 'loc' we can stray at - // this error level. - bin_min = 0; - bin_mid = bin_max; - while (bin_min < bin_mid) { - if (dmp.matchBitapScore(d, loc + bin_mid, loc, pattern) <= threshold) { - bin_min = bin_mid; - } else { - bin_max = bin_mid; - } - bin_mid = (bin_max - bin_min) / 2 + bin_min; - } - // Use the result from this iteration as the maximum for the next. - bin_max = bin_mid; - var start = @max(1, loc - bin_mid + 1); - const finish = @min(loc + bin_mid, text.len) + pattern.len; - var rd = try allocator.alloc(UIntType, finish + 2); - const dshift: ShiftWidth = @intCast(d); - rd[finish + 1] = 1 << dshift; - var j = finish; - while (j >= start) : (j -= 1) { - const char_match: usize = if (text.len <= j - 1 or !map.contains(text[j - 1])) - // Out of range. - 0 - else - map.get(text[j - 1]); - if (d == 0) { - // First pass: exact match. - rd[j] = ((rd[j + 1] << 1)) & char_match; - } else { - // Subsequent passes: fuzzy match. - rd[j] = ((rd[j + 1] << 1)) & char_match & (((last_rd[j + 1] & last_rd[j]) << 1)) & last_rd[j + 1]; - } - if ((rd[j] & matchmask) != 0) { - const score = dmp.matchBitapScore(d, j - 1, loc, pattern); - // This match will almost certainly be better than any existing - // match. But check anyway. - if (score <= threshold) { - // Told you so. - threshold = score; - best_loc = j - 1; - if (best_loc > loc) { - // When passing loc, don't exceed our current distance from loc. - start = @max(1, 2 * loc - best_loc); - } else { - // Already passed loc, downhill from here on in. - break; - } - } - } - } - if (dmp.matchBitapScore(d + 1, loc, loc, pattern) > threshold) { - // No hope for a (better) match at greater error levels. - break; - } - allocator.free(last_rd); - last_rd = rd; - } - allocator.free(last_rd); - return best_loc; -} - -fn ShiftSizeForType(T: type) type { - return switch (@typeInfo(T.Int.bits)) { - 64 => u6, - 256 => u8, - 1024 => u9, - 2048 => u10, - else => unreachable, - }; -} - -const sh_one: u64 = 1; - /// Locate the best instance of `pattern` in `text` near `loc` using the /// Bitap algorithm. Returns -1 if no match found. /// @@ -2315,27 +2180,6 @@ fn matchAlphabet(allocator: Allocator, pattern: []const u8) !std.AutoHashMap(u8, return map; } -/// Initialise the alphabet for the Bitap algorithm. -/// @param pattern The text to encode. -/// @return Hash of character locations. -fn matchAlphabetImproved(allocator: Allocator, pattern: []const u8, UIntSize: type) !std.HashMap(u8, usize) { - const ShiftType = ShiftSizeForType(UIntSize); - var map = std.HashMap(u8, usize).init(allocator); - errdefer map.deinit(); - for (pattern) |c| { - if (!map.contains(c)) { - try map.put(c, 0); - } - } - for (pattern, 0..) |c, i| { - const shift: ShiftType = @intCast(pattern.len - i - 1); - // TODO I think we want c_mask & ~ 1 << shift here: - const value: UIntSize = map.get(c) | (1 << shift); - try map.put(c, value); - } - return map; -} - //| PATCH FUNCTIONS /// Increase the context until it is unique, but don't let the pattern diff --git a/make-file-list.py b/make-file-list.py deleted file mode 100644 index 6f5caab..0000000 --- a/make-file-list.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import git - -# Variables -REPO_PATH = '.' -FILE_NAME = 'DiffMatchPatch.zig' -OUTPUT_DIR = 'file-versions' - -# Initialize the repository -repo = git.Repo(REPO_PATH) - -# Create the output directory if it doesn't exist -if not os.path.exists(OUTPUT_DIR): - os.makedirs(OUTPUT_DIR) - -# Get a list of all commits that modified the file -commits = list(repo.iter_commits(paths=FILE_NAME)) -commits.reverse() - -# Loop through each commit -for i, commit in enumerate(commits): - # Checkout the file from the specific commit - file_content = (repo.git.show(f'{commit.hexsha}:{FILE_NAME}')) - # Write the file content to the output directory with a suffix - with open(os.path.join(OUTPUT_DIR, f'file-{i+1:02d}.zig'), 'w') as f: - f.write(file_content) \ No newline at end of file From 663ae11982a4f3745e8cd9ca6c2847de96f1e611 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 22:38:00 -0400 Subject: [PATCH 159/176] Restore necessary const --- DiffMatchPatch.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 164a0b6..dff35a5 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2002,6 +2002,8 @@ pub fn matchMain( } } +const sh_one: u64 = 1; + /// Locate the best instance of `pattern` in `text` near `loc` using the /// Bitap algorithm. Returns -1 if no match found. /// From 2d373c7d4c25435655127e2f1c1824dfcf80d79b Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 22:43:24 -0400 Subject: [PATCH 160/176] Move utility functions to own section --- DiffMatchPatch.zig | 56 +++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index dff35a5..3478e28 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -801,18 +801,6 @@ fn diffBisectSplit( return diffs; } -inline fn fixSplitForward(text: []const u8, i: usize) usize { - var idx = i; - while (idx < text.len and is_follow(text[idx])) : (idx += 1) {} - return idx; -} - -inline fn fixSplitBackward(text: []const u8, i: usize) usize { - var idx = i; - if (idx < text.len) while (idx != 0 and is_follow(text[idx])) : (idx -= 1) {}; - return idx; -} - /// Do a quick line-level diff on both strings, then rediff the parts for /// greater accuracy. /// This speedup can produce non-minimal diffs. @@ -1582,10 +1570,6 @@ fn diffCleanupSemanticScore(one: []const u8, two: []const u8) usize { return 0; } -inline fn boolInt(b: bool) u8 { - return @intFromBool(b); -} - /// Reduce the number of edits by eliminating operationally trivial /// equalities. pub fn diffCleanupEfficiency( @@ -2452,10 +2436,6 @@ pub fn makePatchFromDiffs(dmp: DiffMatchPatch, allocator: Allocator, diffs: Diff return try dmp.makePatch(allocator, text1, diffs); } -inline fn cast(as: type, val: anytype) as { - return @intCast(val); -} - /// Merge a set of patches onto the text. Returns a tuple: the first of which /// is the patched text, the second of which is... /// @@ -2937,12 +2917,6 @@ pub fn patchFromText(allocator: Allocator, text: []const u8) !PatchList { return patches; } -fn countDigits(text: []const u8) usize { - var idx: usize = 0; - while (std.ascii.isDigit(text[idx])) : (idx += 1) {} - return idx; -} - fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patch } { var patch = Patch{ .diffs = DiffList{} }; errdefer patch.deinit(allocator); @@ -3166,6 +3140,36 @@ fn encodeUri(allocator: std.mem.Allocator, text: []const u8) ![]u8 { return charlist.toOwnedSlice(); } +//| +//| UTILITIES +//| + +inline fn boolInt(b: bool) u8 { + return @intFromBool(b); +} + +inline fn fixSplitForward(text: []const u8, i: usize) usize { + var idx = i; + while (idx < text.len and is_follow(text[idx])) : (idx += 1) {} + return idx; +} + +inline fn fixSplitBackward(text: []const u8, i: usize) usize { + var idx = i; + if (idx < text.len) while (idx != 0 and is_follow(text[idx])) : (idx -= 1) {}; + return idx; +} + +inline fn cast(as: type, val: anytype) as { + return @intCast(val); +} + +fn countDigits(text: []const u8) usize { + var idx: usize = 0; + while (std.ascii.isDigit(text[idx])) : (idx += 1) {} + return idx; +} + //| //| TESTS //| From ea3e231eabe203d9a083f82618fc2e059212b1a5 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 22:49:45 -0400 Subject: [PATCH 161/176] Use boolean to eliminate 19 @intCasts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tidying up a bothersome C♯-ism. --- DiffMatchPatch.zig | 56 +++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 3478e28..66e1cbb 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1268,30 +1268,31 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!void { var changes = false; // Stack of indices where equalities are found. - var equalities = ArrayListUnmanaged(isize){}; + var equalities = ArrayListUnmanaged(usize){}; defer equalities.deinit(allocator); // Always equal to equalities[equalitiesLength-1][1] var last_equality: ?[]const u8 = null; - var pointer: isize = 0; // Index of current position. + var pointer: usize = 0; // Index of current position. // Number of characters that changed prior to the equality. var length_insertions1: usize = 0; var length_deletions1: usize = 0; // Number of characters that changed after the equality. var length_insertions2: usize = 0; var length_deletions2: usize = 0; + var reset_pointer = false; while (pointer < diffs.items.len) { - if (diffs.items[@intCast(pointer)].operation == .equal) { // Equality found. + if (diffs.items[pointer].operation == .equal) { // Equality found. try equalities.append(allocator, pointer); length_insertions1 = length_insertions2; length_deletions1 = length_deletions2; length_insertions2 = 0; length_deletions2 = 0; - last_equality = diffs.items[@intCast(pointer)].text; + last_equality = diffs.items[pointer].text; } else { // an insertion or deletion - if (diffs.items[@intCast(pointer)].operation == .insert) { - length_insertions2 += diffs.items[@intCast(pointer)].text.len; + if (diffs.items[pointer].operation == .insert) { + length_insertions2 += diffs.items[pointer].text.len; } else { - length_deletions2 += diffs.items[@intCast(pointer)].text.len; + length_deletions2 += diffs.items[pointer].text.len; } // Eliminate an equality that is smaller or equal to the edits on both // sides of it. @@ -1302,20 +1303,24 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError // Duplicate record. try diffs.ensureUnusedCapacity(allocator, 1); diffs.insertAssumeCapacity( - @intCast(equalities.items[equalities.items.len - 1]), + equalities.items[equalities.items.len - 1], Diff.init( .delete, try allocator.dupe(u8, last_equality.?), ), ); // Change second copy to insert. - diffs.items[@intCast(equalities.items[equalities.items.len - 1] + 1)].operation = .insert; + diffs.items[equalities.items[equalities.items.len - 1] + 1].operation = .insert; // Throw away the equality we just deleted. _ = equalities.pop(); if (equalities.items.len > 0) { _ = equalities.pop(); } - pointer = if (equalities.items.len > 0) equalities.items[equalities.items.len - 1] else -1; + if (equalities.items.len > 0) { + pointer = equalities.items[equalities.items.len - 1]; + } else { + reset_pointer = true; + } length_insertions1 = 0; // Reset the counters. length_deletions1 = 0; length_insertions2 = 0; @@ -1324,7 +1329,12 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError changes = true; } } - pointer += 1; + if (reset_pointer) { + pointer = 0; + reset_pointer = false; + } else { + pointer += 1; + } } // Normalize the diff. @@ -1341,11 +1351,11 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError // Only extract an overlap if it is as big as the edit ahead or behind it. pointer = 1; while (pointer < diffs.items.len) { - if (diffs.items[@intCast(pointer - 1)].operation == .delete and - diffs.items[@intCast(pointer)].operation == .insert) + if (diffs.items[pointer - 1].operation == .delete and + diffs.items[pointer].operation == .insert) { - const deletion = diffs.items[@intCast(pointer - 1)].text; - const insertion = diffs.items[@intCast(pointer)].text; + const deletion = diffs.items[pointer - 1].text; + const insertion = diffs.items[pointer].text; const overlap_length1: usize = diffCommonOverlap(deletion, insertion); const overlap_length2: usize = diffCommonOverlap(insertion, deletion); if (overlap_length1 >= overlap_length2) { @@ -1356,16 +1366,16 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError // Insert an equality and trim the surrounding edits. try diffs.ensureUnusedCapacity(allocator, 1); diffs.insertAssumeCapacity( - @intCast(pointer), + pointer, Diff.init( .equal, try allocator.dupe(u8, insertion[0..overlap_length1]), ), ); - diffs.items[@intCast(pointer - 1)].text = + diffs.items[pointer - 1].text = try allocator.dupe(u8, deletion[0 .. deletion.len - overlap_length1]); allocator.free(deletion); - diffs.items[@intCast(pointer + 1)].text = + diffs.items[pointer + 1].text = try allocator.dupe(u8, insertion[overlap_length1..]); allocator.free(insertion); pointer += 1; @@ -1378,7 +1388,7 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError // Insert an equality and swap and trim the surrounding edits. try diffs.ensureUnusedCapacity(allocator, 1); diffs.insertAssumeCapacity( - @intCast(pointer), + pointer, Diff.init( .equal, try allocator.dupe(u8, deletion[0..overlap_length2]), @@ -1389,10 +1399,10 @@ fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError const new_plus = try allocator.dupe(u8, deletion[overlap_length2..]); allocator.free(deletion); allocator.free(insertion); - diffs.items[@intCast(pointer - 1)].operation = .insert; - diffs.items[@intCast(pointer - 1)].text = new_minus; - diffs.items[@intCast(pointer + 1)].operation = .delete; - diffs.items[@intCast(pointer + 1)].text = new_plus; + diffs.items[pointer - 1].operation = .insert; + diffs.items[pointer - 1].text = new_minus; + diffs.items[pointer + 1].operation = .delete; + diffs.items[pointer + 1].text = new_plus; pointer += 1; } } From 4ca65aae3b3bfff8a871aad7e982f5f9a923e240 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Fri, 12 Jul 2024 23:05:17 -0400 Subject: [PATCH 162/176] diffLevenshtein returns f64 --- DiffMatchPatch.zig | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 66e1cbb..61ea7a7 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1905,7 +1905,7 @@ pub fn diffAfterText(allocator: Allocator, diffs: DiffList) ![]const u8 { /// @param diffs List of Diff objects. /// @return Number of changes. /// -pub fn diffLevenshtein(diffs: DiffList) usize { +pub fn diffLevenshtein(diffs: DiffList) f64 { var inserts: usize = 0; var deletes: usize = 0; var levenshtein: usize = 0; @@ -1926,7 +1926,7 @@ pub fn diffLevenshtein(diffs: DiffList) usize { } } - return levenshtein + @max(inserts, deletes); + return @floatFromInt(levenshtein + @max(inserts, deletes)); } test diffLevenshtein { @@ -2450,7 +2450,12 @@ pub fn makePatchFromDiffs(dmp: DiffMatchPatch, allocator: Allocator, diffs: Diff /// is the patched text, the second of which is... /// /// TODO I'm just going to return a boolean saying whether all patches -/// were successful. Rethink this at some point. +/// were successful. Rethink this at some point. Possibility: build up a +/// patch string with all unsuccessful patches, it's a legible plain-text +/// format containing the failed edits, which could be converted into a patch +/// again, or used directly in an error message, or the slop turned up on the +/// dmp object and the patch reattempted. The delta allows us to adjust any +/// failed patches so they "fit" the next text. /// /// @param patches Array of Patch objects /// @param text Old text. @@ -2551,10 +2556,8 @@ pub fn patchApply( ); defer deinitDiffList(allocator, &diffs); const t1_l_float: f64 = @floatFromInt(text1.len); - // TODO this is the only place diffLevenshtein gets used, so it - // should just return a float. Probably requires changing the tests. - const levenshtein_float: f64 = @floatFromInt(diffLevenshtein(diffs)); - const bad_match = levenshtein_float / t1_l_float > dmp.patch_delete_threshold; + const levenshtein: f64 = diffLevenshtein(diffs); + const bad_match = levenshtein / t1_l_float > dmp.patch_delete_threshold; if (text1.len > m_max_b and bad_match) { // The end points match, but the content is unacceptably bad. // results[x] = false; From 1d0c79fa2ca0d3e7cf4a5cac16decfdc2eb481da Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sat, 13 Jul 2024 12:19:23 -0400 Subject: [PATCH 163/176] Fix: logic and memory bugs in patchFromHeader I intend to circle back and make all the BadPatchText's check for fails as well. --- DiffMatchPatch.zig | 82 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 9 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 61ea7a7..05cb509 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -2918,7 +2918,7 @@ pub fn writePatch(writer: anytype, patches: PatchList) !void { pub fn patchFromText(allocator: Allocator, text: []const u8) !PatchList { if (text.len == 0) return PatchList{}; var patches = PatchList{}; - errdefer patches.deinit(allocator); + errdefer deinitPatchList(allocator, &patches); var cursor: usize = 0; while (cursor < text.len) { // TODO catch BadPatchString here and print diagnostic @@ -2937,13 +2937,13 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc if (std.mem.eql(u8, text[0..4], PATCH_HEAD)) { // Parse location and length in before text const count = 4 + countDigits(text[4..]); + if (count == 4) return error.BadPatchString; patch.start1 = std.fmt.parseInt( usize, text[4..count], 10, ) catch return error.BadPatchString; cursor = count; - assert(cursor > 4); if (text[cursor] != ',') { patch.start1 -= 1; patch.length1 = 1; @@ -2955,7 +2955,7 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc text[cursor .. cursor + delta], 10, ) catch return error.BadPatchString; - assert(delta > 0); + if (delta == 0) return error.BadPatchString; cursor += delta; if (patch.length1 != 0) { patch.start1 -= 1; @@ -2966,7 +2966,7 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc if (text[cursor] == ' ' and text[cursor + 1] == '+') { cursor += 2; const delta1 = countDigits(text[cursor..]); - assert(delta1 > 0); + if (delta1 == 0) return error.BadPatchString; patch.start2 = std.fmt.parseInt( usize, text[cursor .. cursor + delta1], @@ -2979,7 +2979,7 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc } else { cursor += 1; const delta2 = countDigits(text[cursor..]); - assert(delta2 > 0); + if (delta2 == 0) return error.BadPatchString; patch.length2 = std.fmt.parseInt( usize, text[cursor .. cursor + delta2], @@ -2991,7 +2991,7 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc } } } else return error.BadPatchString; - if (std.mem.eql(u8, text[cursor .. cursor + 4], PATCH_TAIL)) { + if (cursor + 4 <= text.len and std.mem.eql(u8, text[cursor .. cursor + 4], PATCH_TAIL)) { cursor += 4; } else return error.BadPatchString; // Eat the diffs @@ -3044,7 +3044,8 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc }, '@' => { // Start of next patch // back out cursor - cursor -= line.len - 1; + allocator.free(diff_line); + cursor -= line.len + 1; break :patch_loop; }, else => return error.BadPatchString, @@ -4915,7 +4916,7 @@ test diffIndex { } } -test "diffPrettyFormat" { +test diffPrettyFormat { const test_deco = DiffDecorations{ .delete_start = "<+>", .delete_end = "", @@ -5323,6 +5324,13 @@ fn testPatchRoundTrip(allocator: Allocator, patch_in: []const u8) !void { try testing.expectEqualStrings(patch_in, patch_out); } +test "workshop" { + try testPatchRoundTrip( + testing.allocator, + "@@ -0,0 +1,3 @@\n+abc\n@@ -0,0 +1,3 @@\n+abc\n", + ); +} + test "patch from text" { const allocator = testing.allocator; var p0 = try patchFromText(allocator, ""); @@ -5348,7 +5356,63 @@ test "patch from text" { testPatchRoundTrip, .{"@@ -0,0 +1,3 @@\n+abc\n"}, ); - try testing.expectError(error.BadPatchString, patchFromText(allocator, "Bad\nPatch\nString\n")); + try std.testing.checkAllAllocationFailures( + testing.allocator, + testPatchRoundTrip, + .{"@@ -0,0 +1,3 @@\n+abc\n@@ -0,0 +1,3 @@\n+abc\n"}, + ); + try testing.expectError( + error.BadPatchString, + patchFromText(allocator, "Bad\nPatch\nString\n"), + ); + try testing.expectError( + error.BadPatchString, + patchFromText(allocator, "@@ foo"), + ); + try testing.expectError( + error.BadPatchString, + patchFromText(allocator, "@@ +no"), + ); + try testing.expectError( + error.BadPatchString, + patchFromText(allocator, "@@ -no"), + ); + try testing.expectError( + error.BadPatchString, + patchFromText(allocator, "@@ -1,no"), + ); + try testing.expectError( + error.BadPatchString, + patchFromText(allocator, "@@ !1,no"), + ); + try testing.expectError( + error.BadPatchString, + patchFromText(allocator, "@@ -1,3 +???"), + ); + try testing.expectError( + error.BadPatchString, + patchFromText(allocator, "@@ -1,no"), + ); + try testing.expectError( + error.BadPatchString, + patchFromText(allocator, "@@ -1,3 +4,5 ##"), + ); + try testing.expectError( + error.BadPatchString, + patchFromText(allocator, "@@ -1,10??"), + ); + try testing.expectError( + error.BadPatchString, + patchFromText(allocator, "@@ -1,10 ?"), + ); + try testing.expectError( + error.BadPatchString, + patchFromText(allocator, "@@@ -1,3 +4,5 @!"), + ); + try testing.expectError( + error.BadPatchString, + patchFromText(allocator, "@@@ -1,3 +4,5 +add\n@!"), + ); } fn testPatchAddContext( From 42f36f4c68453b8edce5e4016ee3d6fad411f87e Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sat, 13 Jul 2024 13:06:46 -0400 Subject: [PATCH 164/176] Check memory failure within testBadPatchString --- DiffMatchPatch.zig | 110 +++++++++++++++++++++++++++++---------------- 1 file changed, 71 insertions(+), 39 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 05cb509..c28a5b5 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -5361,57 +5361,89 @@ test "patch from text" { testPatchRoundTrip, .{"@@ -0,0 +1,3 @@\n+abc\n@@ -0,0 +1,3 @@\n+abc\n"}, ); - try testing.expectError( - error.BadPatchString, - patchFromText(allocator, "Bad\nPatch\nString\n"), + try std.testing.checkAllAllocationFailures( + testing.allocator, + testBadPatchString, + .{}, ); - try testing.expectError( - error.BadPatchString, - patchFromText(allocator, "@@ foo"), +} + +fn testBadPatchString(allocator: Allocator, patch: []const u8) !void { + _ = patchFromText(allocator, patch) catch |e| { + switch (e) { + error.OutOfMemory => return error.OutOfMemory, + else => { + try testing.expectEqual(error.BadPatchString, e); + }, + } + }; +} + +test "error.BadPatchString" { + try testing.checkAllAllocationFailures( + testing.allocator, + testBadPatchString, + .{"Bad\nPatch\nString\n"}, ); - try testing.expectError( - error.BadPatchString, - patchFromText(allocator, "@@ +no"), + try testing.checkAllAllocationFailures( + testing.allocator, + testBadPatchString, + .{"@@ foo"}, ); - try testing.expectError( - error.BadPatchString, - patchFromText(allocator, "@@ -no"), + try testing.checkAllAllocationFailures( + testing.allocator, + testBadPatchString, + .{"@@ +no"}, ); - try testing.expectError( - error.BadPatchString, - patchFromText(allocator, "@@ -1,no"), + try testing.checkAllAllocationFailures( + testing.allocator, + testBadPatchString, + .{"@@ -no"}, ); - try testing.expectError( - error.BadPatchString, - patchFromText(allocator, "@@ !1,no"), + try testing.checkAllAllocationFailures( + testing.allocator, + testBadPatchString, + .{"@@ -1,no"}, ); - try testing.expectError( - error.BadPatchString, - patchFromText(allocator, "@@ -1,3 +???"), + try testing.checkAllAllocationFailures( + testing.allocator, + testBadPatchString, + .{"@@ !1,no"}, ); - try testing.expectError( - error.BadPatchString, - patchFromText(allocator, "@@ -1,no"), + try testing.checkAllAllocationFailures( + testing.allocator, + testBadPatchString, + .{"@@ -1,3 +???"}, ); - try testing.expectError( - error.BadPatchString, - patchFromText(allocator, "@@ -1,3 +4,5 ##"), + try testing.checkAllAllocationFailures( + testing.allocator, + testBadPatchString, + .{"@@ -1,no"}, + ); + try testing.checkAllAllocationFailures( + testing.allocator, + testBadPatchString, + .{"@@ -1,3 +4,5 ##"}, ); - try testing.expectError( - error.BadPatchString, - patchFromText(allocator, "@@ -1,10??"), + try testing.checkAllAllocationFailures( + testing.allocator, + testBadPatchString, + .{"@@ -1,10??"}, ); - try testing.expectError( - error.BadPatchString, - patchFromText(allocator, "@@ -1,10 ?"), + try testing.checkAllAllocationFailures( + testing.allocator, + testBadPatchString, + .{"@@ -1,10 ?"}, ); - try testing.expectError( - error.BadPatchString, - patchFromText(allocator, "@@@ -1,3 +4,5 @!"), + try testing.checkAllAllocationFailures( + testing.allocator, + testBadPatchString, + .{"@@@ -1,3 +4,5 @!"}, ); - try testing.expectError( - error.BadPatchString, - patchFromText(allocator, "@@@ -1,3 +4,5 +add\n@!"), + try testing.checkAllAllocationFailures( + testing.allocator, + testBadPatchString, + .{"@@@ -1,3 +4,5 +add\n@!"}, ); } From e01668b4801662fcdd6333109ea8fdeddf576b37 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sat, 13 Jul 2024 14:44:36 -0400 Subject: [PATCH 165/176] Add post-first-patch failing case --- DiffMatchPatch.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index c28a5b5..9dbefb6 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -5361,11 +5361,6 @@ test "patch from text" { testPatchRoundTrip, .{"@@ -0,0 +1,3 @@\n+abc\n@@ -0,0 +1,3 @@\n+abc\n"}, ); - try std.testing.checkAllAllocationFailures( - testing.allocator, - testBadPatchString, - .{}, - ); } fn testBadPatchString(allocator: Allocator, patch: []const u8) !void { @@ -5445,6 +5440,11 @@ test "error.BadPatchString" { testBadPatchString, .{"@@@ -1,3 +4,5 +add\n@!"}, ); + try std.testing.checkAllAllocationFailures( + testing.allocator, + testBadPatchString, + .{"@@ -0,0 +1,3 @@\n+abc\n@@ -0,0 +1,3 @@\n+abc\n!!!"}, + ); } fn testPatchAddContext( From ba58e7e74659ba14a0238a59c305c9ef9e2ce6cd Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sat, 13 Jul 2024 19:30:48 -0400 Subject: [PATCH 166/176] Tracking down line-mode fails --- DiffMatchPatch.zig | 75 +++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 30 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 9dbefb6..ea7f33b 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -823,7 +823,7 @@ fn diffLineMode( const line_array = a.line_array; var diffs: DiffList = try dmp.diffInternal(allocator, text1, text2, false, deadline); - errdefer diffs.deinit(allocator); + errdefer deinitDiffList(allocator, &diffs); // Convert the diff back to original text. try diffCharsToLines(allocator, diffs.items, line_array.items); // Eliminate freak matches (e.g. blank lines) @@ -928,6 +928,7 @@ fn diffLinesToChars( // Allocate 2/3rds of the space for text1, the rest for text2. const chars1 = try diffLinesToCharsMunge(allocator, text1, &line_array, &line_hash, UNICODE_TWO_THIRDS); + errdefer allocator.free(chars1); const chars2 = try diffLinesToCharsMunge(allocator, text2, &line_array, &line_hash, UNICODE_ONE_THIRD); return .{ .chars_1 = chars1, .chars_2 = chars2, .line_array = line_array }; } @@ -4003,11 +4004,11 @@ fn testDiff( } test diff { - const this: DiffMatchPatch = .{ .diff_timeout = 0 }; + const dmp: DiffMatchPatch = .{ .diff_timeout = 0 }; // Null case. try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ - .dmp = this, + .dmp = dmp, .before = "", .after = "", .check_lines = false, @@ -4016,7 +4017,7 @@ test diff { // Equality. try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ - .dmp = this, + .dmp = dmp, .before = "abc", .after = "abc", .check_lines = false, @@ -4027,7 +4028,7 @@ test diff { // Simple insertion. try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ - .dmp = this, + .dmp = dmp, .before = "abc", .after = "ab123c", .check_lines = false, @@ -4040,7 +4041,7 @@ test diff { // Simple deletion. try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ - .dmp = this, + .dmp = dmp, .before = "a123bc", .after = "abc", .check_lines = false, @@ -4053,7 +4054,7 @@ test diff { // Two insertions. try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ - .dmp = this, + .dmp = dmp, .before = "abc", .after = "a123b456c", .check_lines = false, @@ -4068,7 +4069,7 @@ test diff { // Two deletions. try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ - .dmp = this, + .dmp = dmp, .before = "a123b456c", .after = "abc", .check_lines = false, @@ -4083,7 +4084,7 @@ test diff { // Simple case #1 try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ - .dmp = this, + .dmp = dmp, .before = "a", .after = "b", .check_lines = false, @@ -4095,7 +4096,7 @@ test diff { // Simple case #2 try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ - .dmp = this, + .dmp = dmp, .before = "Apples are a fruit.", .after = "Bananas are also fruit.", .check_lines = false, @@ -4110,7 +4111,7 @@ test diff { // Simple case #3 try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ - .dmp = this, + .dmp = dmp, .before = "ax\t", .after = "\u{0680}x\x00", .check_lines = false, @@ -4125,7 +4126,7 @@ test diff { // Overlap #1 try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ - .dmp = this, + .dmp = dmp, .before = "1ayb2", .after = "abxab", .check_lines = false, @@ -4141,7 +4142,7 @@ test diff { // Overlap #2 try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ - .dmp = this, + .dmp = dmp, .before = "abcy", .after = "xaxcxabc", .check_lines = false, @@ -4154,7 +4155,7 @@ test diff { // Overlap #3 try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ - .dmp = this, + .dmp = dmp, .before = "ABCDa=bcd=efghijklmnopqrsEFGHIJKLMNOefg", .after = "a-bcd-efghijklmnopqrs", .check_lines = false, @@ -4173,7 +4174,7 @@ test diff { // Large equality try testing.checkAllAllocationFailures(testing.allocator, testDiff, .{.{ - .dmp = this, + .dmp = dmp, .before = "a [[Pennsylvania]] and [[New", .after = " and [[Pennsylvania]]", .check_lines = false, @@ -4212,30 +4213,44 @@ test diff { // OS task swaps or locks up for a second at the wrong moment. try testing.expect((with_timout.diff_timeout) * 10000 * 2 > end_time - start_time); // diff: Timeout max. } +} - { - // Test the linemode speedup. - // Must be long to pass the 100 char cutoff. - const a = "1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n"; - const b = "abcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\n"; +fn testDiffLineMode( + allocator: Allocator, + dmp: DiffMatchPatch, + before: []const u8, + after: []const u8, +) !void { + var diff_checked = try dmp.diff(allocator, before, after, true); + defer deinitDiffList(allocator, &diff_checked); - var diff_checked = try this.diff(allocator, a, b, true); - defer deinitDiffList(allocator, &diff_checked); + var diff_unchecked = try dmp.diff(allocator, before, after, false); + defer deinitDiffList(allocator, &diff_unchecked); - var diff_unchecked = try this.diff(allocator, a, b, false); - defer deinitDiffList(allocator, &diff_unchecked); + try testing.expectEqualDeep(diff_checked.items, diff_unchecked.items); // diff: Simple line-mode. +} - try testing.expectEqualDeep(diff_checked.items, diff_unchecked.items); // diff: Simple line-mode. - } +test "diffLineMode" { + const dmp: DiffMatchPatch = .{ .diff_timeout = 0 }; + const allocator = testing.allocator; + try testing.checkAllAllocationFailures( + testing.allocator, + testDiffLineMode, + + .{ + dmp, "1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n", + "abcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\n", + }, + ); { const a = "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"; const b = "abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij"; - var diff_checked = try this.diff(allocator, a, b, true); + var diff_checked = try dmp.diff(allocator, a, b, true); defer deinitDiffList(allocator, &diff_checked); - var diff_unchecked = try this.diff(allocator, a, b, false); + var diff_unchecked = try dmp.diff(allocator, a, b, false); defer deinitDiffList(allocator, &diff_unchecked); try testing.expectEqualDeep(diff_checked.items, diff_unchecked.items); // diff: Single line-mode. @@ -4246,7 +4261,7 @@ test diff { const a = "1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n"; const b = "abcdefghij\n1234567890\n1234567890\n1234567890\nabcdefghij\n1234567890\n1234567890\n1234567890\nabcdefghij\n1234567890\n1234567890\n1234567890\nabcdefghij\n"; - var diffs_linemode = try this.diff(allocator, a, b, true); + var diffs_linemode = try dmp.diff(allocator, a, b, true); defer deinitDiffList(allocator, &diffs_linemode); const texts_linemode = try rebuildtexts(allocator, diffs_linemode); @@ -4255,7 +4270,7 @@ test diff { allocator.free(texts_linemode[1]); } - var diffs_textmode = try this.diff(allocator, a, b, false); + var diffs_textmode = try dmp.diff(allocator, a, b, false); defer deinitDiffList(allocator, &diffs_textmode); const texts_textmode = try rebuildtexts(allocator, diffs_textmode); From f22ccb48a27bea4d643b6b9b7c174df261725662 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sat, 13 Jul 2024 20:36:28 -0400 Subject: [PATCH 167/176] Port fix for line mode from memory branch --- DiffMatchPatch.zig | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index ea7f33b..c8d9f4b 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -821,11 +821,14 @@ fn diffLineMode( const text1 = a.chars_1; const text2 = a.chars_2; const line_array = a.line_array; - - var diffs: DiffList = try dmp.diffInternal(allocator, text1, text2, false, deadline); + var diffs: DiffList = undefined; + { + var char_diffs: DiffList = try dmp.diffInternal(allocator, text1, text2, false, deadline); + defer deinitDiffList(allocator, &char_diffs); + // Convert the diff back to original text. + diffs = try diffCharsToLines(allocator, &char_diffs, line_array.items); + } errdefer deinitDiffList(allocator, &diffs); - // Convert the diff back to original text. - try diffCharsToLines(allocator, diffs.items, line_array.items); // Eliminate freak matches (e.g. blank lines) try diffCleanupSemantic(allocator, &diffs); @@ -877,8 +880,13 @@ fn diffLineMode( false, deadline, ); + { + errdefer deinitDiffList(allocator, &sub_diff); + try diffs.ensureUnusedCapacity(allocator, sub_diff.items.len); + } defer sub_diff.deinit(allocator); - try diffs.insertSlice(allocator, pointer, sub_diff.items); + const new_diff = diffs.addManyAtAssumeCapacity(pointer, sub_diff.items.len); + @memcpy(new_diff, sub_diff.items); pointer = pointer + sub_diff.items.len; } count_insert = 0; @@ -1032,12 +1040,15 @@ fn diffIteratorToCharsMunge( /// @param lineArray List of unique strings. fn diffCharsToLines( allocator: Allocator, - diffs: []Diff, + char_diffs: *DiffList, line_array: []const []const u8, -) DiffError!void { +) DiffError!DiffList { var text = ArrayListUnmanaged(u8){}; defer text.deinit(allocator); - for (diffs) |*d| { + var diffs = DiffList{}; + errdefer deinitDiffList(allocator, &diffs); + try diffs.ensureUnusedCapacity(allocator, char_diffs.items.len); + for (char_diffs.items) |*d| { var cursor: usize = 0; while (cursor < d.text.len) { const cp_len = std.unicode.utf8ByteSequenceLength(d.text[cursor]) catch { @@ -1049,9 +1060,12 @@ fn diffCharsToLines( try text.appendSlice(allocator, line_array[cp - CHAR_OFFSET]); cursor += cp_len; } - allocator.free(d.text); - d.text = try text.toOwnedSlice(allocator); + diffs.appendAssumeCapacity(Diff.init( + d.operation, + try text.toOwnedSlice(allocator), + )); } + return diffs; } /// An iteration struct over lines, which includes the newline if present. @@ -3486,14 +3500,15 @@ fn testDiffCharsToLines( expected: []const Diff, }, ) !void { - var diffs = try DiffList.initCapacity(allocator, params.diffs.len); - defer deinitDiffList(allocator, &diffs); + var char_diffs = try DiffList.initCapacity(allocator, params.diffs.len); + defer deinitDiffList(allocator, &char_diffs); for (params.diffs) |item| { - diffs.appendAssumeCapacity(.{ .operation = item.operation, .text = try allocator.dupe(u8, item.text) }); + char_diffs.appendAssumeCapacity(.{ .operation = item.operation, .text = try allocator.dupe(u8, item.text) }); } - try diffCharsToLines(allocator, diffs.items, params.line_array); + var diffs = try diffCharsToLines(allocator, &char_diffs, params.line_array); + defer deinitDiffList(allocator, &diffs); try testing.expectEqualDeep(params.expected, diffs.items); } From 85a84b726c3ac3d1c2e817abdb4422455df9a3f2 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sat, 13 Jul 2024 21:13:19 -0400 Subject: [PATCH 168/176] Add configurable line threshold --- DiffMatchPatch.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index c8d9f4b..4672958 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -23,6 +23,9 @@ diff_timeout: u64 = 1000, /// Cost of an empty edit operation in terms of edit characters. diff_edit_cost: u16 = 4, +/// Number of bytes in each string needed to trigger a line-based diff +diff_check_lines_over: u64 = 100, + /// At what point is no match declared (0.0 = perfection, 1.0 = very loose). /// This defaults to 0.05, on the premise that the library will mostly be /// used in cases where failure is better than a bad patch application. @@ -448,7 +451,7 @@ fn diffCompute( return diffs; } - if (check_lines and before.len > 100 and after.len > 100) { + if (check_lines and before.len > dmp.diff_check_lines_over and after.len > dmp.diff_check_lines_over) { return dmp.diffLineMode(allocator, before, after, deadline); } return dmp.diffBisect(allocator, before, after, deadline); From 5d3a83f388676e9b7ba616f275a50b6ed2defacf Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 14 Jul 2024 14:09:53 -0400 Subject: [PATCH 169/176] Add check-lines limit to DiffMatchPatch struct This brings the line-mode allocation failure test down to a point where runtimes are comparable to other tests. --- DiffMatchPatch.zig | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 4672958..fd68f09 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -913,10 +913,12 @@ const UNICODE_MAX = 0x10ffdf; const UNICODE_TWO_THIRDS = 742724; const UNICODE_ONE_THIRD = 371355; const CHAR_OFFSET = 32; + comptime { assert(UNICODE_TWO_THIRDS + UNICODE_ONE_THIRD == UNICODE_MAX); assert(UNICODE_TWO_THIRDS + UNICODE_ONE_THIRD + CHAR_OFFSET == 0x10ffff); } + /// Split two texts into a list of strings. Reduce the texts to a string of /// hashes where each Unicode character represents one line. /// @param text1 First string. @@ -4235,10 +4237,11 @@ test diff { fn testDiffLineMode( allocator: Allocator, - dmp: DiffMatchPatch, + dmp: *DiffMatchPatch, before: []const u8, after: []const u8, ) !void { + dmp.diff_check_lines_over = 20; var diff_checked = try dmp.diff(allocator, before, after, true); defer deinitDiffList(allocator, &diff_checked); @@ -4246,18 +4249,19 @@ fn testDiffLineMode( defer deinitDiffList(allocator, &diff_unchecked); try testing.expectEqualDeep(diff_checked.items, diff_unchecked.items); // diff: Simple line-mode. + dmp.diff_check_lines_over = 100; } test "diffLineMode" { - const dmp: DiffMatchPatch = .{ .diff_timeout = 0 }; + var dmp: DiffMatchPatch = .{ .diff_timeout = 0 }; const allocator = testing.allocator; try testing.checkAllAllocationFailures( testing.allocator, testDiffLineMode, - .{ - dmp, "1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n1234567890\n", - "abcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\nabcdefghij\n", + &dmp, + "1234567890\n1234567890\n1234567890", + "abcdefghij\nabcdefghij\nabcdefghij", }, ); From 1fd9187ffd2d6018fa22ab1b9f7ef9227cc6bcdb Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 14 Jul 2024 14:11:01 -0400 Subject: [PATCH 170/176] Roadmap updates --- roadmap.md | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/roadmap.md b/roadmap.md index e33a91f..d361f70 100644 --- a/roadmap.md +++ b/roadmap.md @@ -13,30 +13,52 @@ - [ ] Refactor: the port currently treats Diffs and Patches as raw ArrayLists, these should be proper Zig objects, with member functions, and probably come in an Unmanaged and normal form. - - [?] Diff struct becomes Edit. + - [?] Diff struct becomes Edit. Patch also needs a name, because Diff and + Patch should be the names of the user-facing structs. The name for + what a patch is in classic diff/patch programs is Hunk, so that's a + justifiable choice. - [ ] DiffList and PatchList remain same, used internally. - [ ] New Diff struct, and DiffUnmanaged. - [ ] Namespaces subsequent operations on diffs. + - [ ] Same for Patch and PatchUnmanaged. These are little more than the + relevant DiffList, a DiffMatchPatch instance, and some decl functions, + plus the Allocator for managed versions. - [ ] Enhancements - [ ] Extend Bitap algorithm to handle larger patches. The algorithm takes `m * n` space, where `m` is unique bytes in the pattern and `n` is the pattern length, so I think the idea of doing it up to 2048 bytes/bits - was optimistic on my part. + was optimistic on my part. But comptime-gated function specializations + for 64 (status quo), 128, and 256 bytes, would mean a lot less frobbing + and munging the patches internally. Performance implications expected + to be positive, if not hugely so. The algorithm is also amenable to + SIMD acceleration, although I'm not going to do that. - [ ] `diffsForRegion`: provides every diff pertaining to a specific region of `before`. Needs to also include how much overlap, if any, the diff includes. Should have "borrow" and "copy" - versions. - - [ ] Implement a delta function which doesn't suck so badly. - - [ ] Diff stream - - [ ] Use Unicode characters and codepoint indices - 32. - - [ ] Implement line diff as a stream. - - [ ] Also gives word diff, token diff, etc. + versions. Signature being `diffsForRegion(diffs: DiffList, start: usize,` + `end: usize, ) ?DiffList`. + - [ ] Implement a delta format which doesn't suck so badly. I have copious + notes on this. + - [ ] I'd also like to break compatibility with the 'Unidiff' format, in a + less dramatic way. It's mostly the compulsive percent-encoding of + everything which doesn't fit in a URI, it's Googley [derogatory] and + a UTF-8 native patch format has no need for this. This would be a + separate, sucks-less text format, differentiable by its header, + decodes into a Patch in the same basic way. The legacy form is + already ported and should be kept. + - [ ] Add `Diff.differs() bool`, which checks if there are any differences + between the before and after text. + - [✅] Diff stream + - [✅] Use Unicode characters and codepoint indices - 32. + - [✅] Implement line diff as a stream. + - [✅] Also gives word diff, token diff, etc. - [ ] Histogram? - [ ] Imara diff has an optimized histogram: https://github.com/pascalkuthe/imara-diff - [ ] Calculating the histogram while hashing the lines would be straightforward, this could be comptime-gated, but probably just a second copy of the munge function is fine. + - [ ] This one is getting into overkill territory perhaps. - [ ] POSIX-diff compatible patch output? - [ ] This one seems pretty worthwhile to me. It would need to call line mode without refining further, but everything else is fairly simple. From bf22ea4d1be878063595f3b6cadd70d4e14c0b0c Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 14 Jul 2024 16:13:28 -0400 Subject: [PATCH 171/176] Make diffCleanupSemantic public --- DiffMatchPatch.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index fd68f09..51fb6bb 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1285,7 +1285,7 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo /// Reduce the number of edits by eliminating semantically trivial /// equalities. /// @param diffs List of Diff objects. -fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!void { +pub fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!void { var changes = false; // Stack of indices where equalities are found. var equalities = ArrayListUnmanaged(usize){}; From bb9a1ae4e7220e5360024df61c369d51516c6860 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 21 Jul 2024 17:14:52 -0400 Subject: [PATCH 172/176] Remove if (false) branches and XXX bool --- DiffMatchPatch.zig | 94 ++++++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 50 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 51fb6bb..b18be73 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -13,9 +13,6 @@ pub const DiffError = error{ BadPatchString, }; -//| XXX This boolean is entirely for calming the compiler down while working -const XXX = false; - //| Fields /// Number of milliseconds to map a diff before giving up (0 for infinity). @@ -5649,51 +5646,49 @@ fn testPatchSplitMax(allocator: Allocator) !void { var dmp = DiffMatchPatch{}; // TODO get some tests which cover the max split we actually use: bitsize(usize) dmp.match_max_bits = 32; - if (false) { - { - var patches = try dmp.diffAndMakePatch( - allocator, - "abcdefghijklmnopqrstuvwxyz01234567890", - "XabXcdXefXghXijXklXmnXopXqrXstXuvXwxXyzX01X23X45X67X89X0", - ); - defer deinitPatchList(allocator, &patches); - const expected_patch = "@@ -1,32 +1,46 @@\n+X\n ab\n+X\n cd\n+X\n ef\n+X\n gh\n+X\n ij\n+X\n kl\n+X\n mn\n+X\n op\n+X\n qr\n+X\n st\n+X\n uv\n+X\n wx\n+X\n yz\n+X\n 012345\n@@ -25,13 +39,18 @@\n zX01\n+X\n 23\n+X\n 45\n+X\n 67\n+X\n 89\n+X\n 0\n"; - try dmp.patchSplitMax(allocator, &patches); - const patch_text = try patchToText(allocator, patches); - defer allocator.free(patch_text); - try testing.expectEqualStrings(expected_patch, patch_text); - } - { - var patches = try dmp.diffAndMakePatch( - allocator, - "abcdef1234567890123456789012345678901234567890123456789012345678901234567890uvwxyz", - "abcdefuvwxyz", - ); - defer deinitPatchList(allocator, &patches); - const text_before = try patchToText(allocator, patches); - defer allocator.free(text_before); - try dmp.patchSplitMax(allocator, &patches); - const text_after = try patchToText(allocator, patches); - defer allocator.free(text_after); - try testing.expectEqualStrings(text_before, text_after); - } - { - var patches = try dmp.diffAndMakePatch( - allocator, - "1234567890123456789012345678901234567890123456789012345678901234567890", - "abc", - ); - defer deinitPatchList(allocator, &patches); - const pre_patch_text = try patchToText(allocator, patches); - defer allocator.free(pre_patch_text); - try dmp.patchSplitMax(allocator, &patches); - const patch_text = try patchToText(allocator, patches); - defer allocator.free(patch_text); - try testing.expectEqualStrings( - "@@ -1,32 +1,4 @@\n-1234567890123456789012345678\n 9012\n@@ -29,32 +1,4 @@\n-9012345678901234567890123456\n 7890\n@@ -57,14 +1,3 @@\n-78901234567890\n+abc\n", - patch_text, - ); - } + { + var patches = try dmp.diffAndMakePatch( + allocator, + "abcdefghijklmnopqrstuvwxyz01234567890", + "XabXcdXefXghXijXklXmnXopXqrXstXuvXwxXyzX01X23X45X67X89X0", + ); + defer deinitPatchList(allocator, &patches); + const expected_patch = "@@ -1,32 +1,46 @@\n+X\n ab\n+X\n cd\n+X\n ef\n+X\n gh\n+X\n ij\n+X\n kl\n+X\n mn\n+X\n op\n+X\n qr\n+X\n st\n+X\n uv\n+X\n wx\n+X\n yz\n+X\n 012345\n@@ -25,13 +39,18 @@\n zX01\n+X\n 23\n+X\n 45\n+X\n 67\n+X\n 89\n+X\n 0\n"; + try dmp.patchSplitMax(allocator, &patches); + const patch_text = try patchToText(allocator, patches); + defer allocator.free(patch_text); + try testing.expectEqualStrings(expected_patch, patch_text); + } + { + var patches = try dmp.diffAndMakePatch( + allocator, + "abcdef1234567890123456789012345678901234567890123456789012345678901234567890uvwxyz", + "abcdefuvwxyz", + ); + defer deinitPatchList(allocator, &patches); + const text_before = try patchToText(allocator, patches); + defer allocator.free(text_before); + try dmp.patchSplitMax(allocator, &patches); + const text_after = try patchToText(allocator, patches); + defer allocator.free(text_after); + try testing.expectEqualStrings(text_before, text_after); + } + { + var patches = try dmp.diffAndMakePatch( + allocator, + "1234567890123456789012345678901234567890123456789012345678901234567890", + "abc", + ); + defer deinitPatchList(allocator, &patches); + const pre_patch_text = try patchToText(allocator, patches); + defer allocator.free(pre_patch_text); + try dmp.patchSplitMax(allocator, &patches); + const patch_text = try patchToText(allocator, patches); + defer allocator.free(patch_text); + try testing.expectEqualStrings( + "@@ -1,32 +1,4 @@\n-1234567890123456789012345678\n 9012\n@@ -29,32 +1,4 @@\n-9012345678901234567890123456\n 7890\n@@ -57,14 +1,3 @@\n-78901234567890\n+abc\n", + patch_text, + ); } { var patches = try dmp.diffAndMakePatch( @@ -5738,9 +5733,8 @@ fn testPatchAddPadding( allocator.free(codes); const patch_text_after = try patchToText(allocator, patches); defer allocator.free(patch_text_after); - if (false) try testing.expectEqualStrings(expect_after, patch_text_after); + try testing.expectEqualStrings(expect_after, patch_text_after); } - test patchAddPadding { // Both edges full. try testing.checkAllAllocationFailures( From 3a1b7effc26b8340fcdcf4fda6892400f5ec9b6d Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Sun, 21 Jul 2024 20:34:00 -0400 Subject: [PATCH 173/176] Change most error sets to error{OutOfMemory} It doesn't make a lot of sense for functions where that's the only case to handle to use a larger superset. --- DiffMatchPatch.zig | 74 +++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index b18be73..4f99198 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -13,6 +13,8 @@ pub const DiffError = error{ BadPatchString, }; +const OutOfMemory = error.OutOfMemory; + //| Fields /// Number of milliseconds to map a diff before giving up (0 for infinity). @@ -223,7 +225,7 @@ pub fn diff( /// to identify the changed areas. If true, then run /// a faster slightly less optimal diff. check_lines: bool, -) DiffError!DiffList { +) error{OutOfMemory}!DiffList { const deadline = if (dmp.diff_timeout == 0) std.math.maxInt(u64) else @@ -238,7 +240,7 @@ fn diffInternal( after: []const u8, check_lines: bool, deadline: u64, -) DiffError!DiffList { +) error{OutOfMemory}!DiffList { // Check for equality (speedup). if (std.mem.eql(u8, before, after)) { var diffs = DiffList{}; @@ -339,7 +341,7 @@ fn diffCompute( after: []const u8, check_lines: bool, deadline: u64, -) DiffError!DiffList { +) error{OutOfMemory}!DiffList { if (before.len == 0) { // Just add some text (speedup). var diffs = DiffList{}; @@ -484,7 +486,7 @@ fn diffHalfMatch( allocator: std.mem.Allocator, before: []const u8, after: []const u8, -) DiffError!?HalfMatchResult { +) error{OutOfMemory}!?HalfMatchResult { if (dmp.diff_timeout == 0) { // Don't risk returning a non-optimal diff if we have unlimited time. return null; @@ -557,7 +559,7 @@ fn diffHalfMatchInternal( long_text: []const u8, short_text: []const u8, i: usize, -) DiffError!?HalfMatchResult { +) error{OutOfMemory}!?HalfMatchResult { // Start with a 1/4 length Substring at position i as a seed. const seed = long_text[i .. i + long_text.len / 4]; var j: isize = -1; @@ -623,7 +625,7 @@ fn diffBisect( before: []const u8, after: []const u8, deadline: u64, -) DiffError!DiffList { +) error{OutOfMemory}!DiffList { const before_length: isize = @intCast(before.len); const after_length: isize = @intCast(after.len); const max_d: isize = @intCast((before.len + after.len + 1) / 2); @@ -778,7 +780,7 @@ fn diffBisectSplit( x: isize, y: isize, deadline: u64, -) DiffError!DiffList { +) error{OutOfMemory}!DiffList { const x1 = fixSplitForward(text1, @intCast(x)); const y1 = fixSplitBackward(text2, @intCast(y)); const text1a = text1[0..x1]; @@ -814,7 +816,7 @@ fn diffLineMode( text1_in: []const u8, text2_in: []const u8, deadline: u64, -) DiffError!DiffList { +) error{OutOfMemory}!DiffList { // Scan the text on a line-by-line basis first. var a = try diffLinesToChars(allocator, text1_in, text2_in); defer a.deinit(allocator); @@ -927,7 +929,7 @@ fn diffLinesToChars( allocator: std.mem.Allocator, text1: []const u8, text2: []const u8, -) DiffError!LinesToCharsResult { +) error{OutOfMemory}!LinesToCharsResult { var line_array = ArrayListUnmanaged([]const u8){}; errdefer line_array.deinit(allocator); line_array.items.len = 0; @@ -968,7 +970,7 @@ fn diffLinesToCharsMunge( line_array: *ArrayListUnmanaged([]const u8), line_hash: *std.StringHashMapUnmanaged(u21), max_lines: usize, -) DiffError![]const u8 { +) error{OutOfMemory}![]const u8 { var iter = LineIterator{ .text = text }; return try diffIteratorToCharsMunge( allocator, @@ -1004,7 +1006,7 @@ fn diffIteratorToCharsMunge( segment_hash: *std.StringHashMapUnmanaged(u21), iterator: anytype, max_segments: usize, -) DiffError![]const u8 { +) error{OutOfMemory}![]const u8 { // Because we rebase the codepoint off the already counted segments, // this makes the unreachables in the function legitimate: assert(max_segments <= UNICODE_MAX); @@ -1044,7 +1046,7 @@ fn diffCharsToLines( allocator: Allocator, char_diffs: *DiffList, line_array: []const []const u8, -) DiffError!DiffList { +) error{OutOfMemory}!DiffList { var text = ArrayListUnmanaged(u8){}; defer text.deinit(allocator); var diffs = DiffList{}; @@ -1107,7 +1109,7 @@ const LineIterator = struct { /// Reorder and merge like edit sections. Merge equalities. /// Any edit section can move as long as it doesn't cross an equality. /// @param diffs List of Diff objects. -fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!void { +fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) error{OutOfMemory}!void { // Add a dummy entry at the end. try diffs.append(allocator, Diff.init(.equal, "")); var pointer: usize = 0; @@ -1282,7 +1284,7 @@ fn diffCleanupMerge(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!vo /// Reduce the number of edits by eliminating semantically trivial /// equalities. /// @param diffs List of Diff objects. -pub fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffError!void { +pub fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) error{OutOfMemory}!void { var changes = false; // Stack of indices where equalities are found. var equalities = ArrayListUnmanaged(usize){}; @@ -1435,7 +1437,7 @@ pub fn diffCleanupSemantic(allocator: std.mem.Allocator, diffs: *DiffList) DiffE pub fn diffCleanupSemanticLossless( allocator: std.mem.Allocator, diffs: *DiffList, -) DiffError!void { +) error{OutOfMemory}!void { var pointer: usize = 1; // Intentionally ignore the first and last element (don't need checking). while (pointer < @as(isize, @intCast(diffs.items.len)) - 1) { @@ -1603,7 +1605,7 @@ pub fn diffCleanupEfficiency( dmp: DiffMatchPatch, allocator: std.mem.Allocator, diffs: *DiffList, -) DiffError!void { +) error{OutOfMemory}!void { var changes = false; // Stack of indices where equalities are found. var equalities = std.ArrayList(usize).init(allocator); @@ -1888,7 +1890,7 @@ pub fn writeDiffPrettyFormat( /// @param diffs List of `Diff` objects. /// @return Source text. /// -pub fn diffBeforeText(allocator: Allocator, diffs: DiffList) ![]const u8 { +pub fn diffBeforeText(allocator: Allocator, diffs: DiffList) error{OutOfMemory}![]const u8 { var chars = ArrayListUnmanaged(u8){}; defer chars.deinit(allocator); for (diffs.items) |d| { @@ -1904,7 +1906,7 @@ pub fn diffBeforeText(allocator: Allocator, diffs: DiffList) ![]const u8 { /// @param diffs List of `Diff` objects. /// @return Destination text. /// -pub fn diffAfterText(allocator: Allocator, diffs: DiffList) ![]const u8 { +pub fn diffAfterText(allocator: Allocator, diffs: DiffList) error{OutOfMemory}![]const u8 { var chars = ArrayListUnmanaged(u8){}; defer chars.deinit(allocator); for (diffs.items) |d| { @@ -1995,7 +1997,7 @@ pub fn matchMain( text: []const u8, pattern: []const u8, passed_loc: usize, -) !?usize { +) error{OutOfMemory}!?usize { // Clamp the loc to fit within text. const loc = @min(passed_loc, text.len); if (std.mem.eql(u8, text, pattern)) { @@ -2028,7 +2030,7 @@ fn matchBitap( text: []const u8, pattern: []const u8, loc: usize, -) !?usize { +) error{OutOfMemory}!?usize { // TODO decide what to do here: // assert (Match_MaxBits == 0 || pattern.Length <= Match_MaxBits) // : "Pattern too long for this application."; @@ -2177,7 +2179,7 @@ fn matchBitapScore( /// Initialise the alphabet for the Bitap algorithm. /// @param pattern The text to encode. /// @return Hash of character locations. -fn matchAlphabet(allocator: Allocator, pattern: []const u8) !std.AutoHashMap(u8, usize) { +fn matchAlphabet(allocator: Allocator, pattern: []const u8) error{OutOfMemory}!std.AutoHashMap(u8, usize) { var map = std.AutoHashMap(u8, usize).init(allocator); errdefer map.deinit(); for (pattern) |c| { @@ -2205,7 +2207,7 @@ fn patchAddContext( allocator: Allocator, patch: *Patch, text: []const u8, -) !void { +) error{OutOfMemory}!void { if (text.len == 0) return; // TODO the fixup logic here might make patterns too large? // It should be ok, because big patches get broken up. Hmm. @@ -2284,7 +2286,7 @@ pub fn diffAndMakePatch( allocator: Allocator, text1: []const u8, text2: []const u8, -) !PatchList { +) error{OutOfMemory}!PatchList { var diffs = try dmp.diff(allocator, text1, text2, true); defer deinitDiffList(allocator, &diffs); if (diffs.items.len > 2) { @@ -2301,7 +2303,7 @@ fn makePatchInternal( text: []const u8, diffs: DiffList, diff_act: DiffHandling, -) !PatchList { +) error{OutOfMemory}!PatchList { var patches = PatchList{}; errdefer deinitPatchList(allocator, &patches); if (diffs.items.len == 0) { @@ -2453,11 +2455,15 @@ pub fn makePatch( allocator: Allocator, text: []const u8, diffs: DiffList, -) !PatchList { +) error{OutOfMemory}!PatchList { return try dmp.makePatchInternal(allocator, text, diffs, .copy); } -pub fn makePatchFromDiffs(dmp: DiffMatchPatch, allocator: Allocator, diffs: DiffList) !PatchList { +pub fn makePatchFromDiffs( + dmp: DiffMatchPatch, + allocator: Allocator, + diffs: DiffList, +) error{OutOfMemory}!PatchList { const text1 = try diffBeforeText(allocator, diffs); defer allocator.free(text1); return try dmp.makePatch(allocator, text1, diffs); @@ -2483,7 +2489,7 @@ pub fn patchApply( allocator: Allocator, og_patches: *PatchList, og_text: []const u8, -) !struct { []const u8, bool } { +) error{OutOfMemory}!struct { []const u8, bool } { if (og_patches.items.len == 0) { // As silly as this is, we dupe the text, because something // passing an empty patchset isn't going to check, and will @@ -2625,7 +2631,7 @@ fn patchSplitMax( dmp: DiffMatchPatch, allocator: Allocator, patches: *PatchList, -) !void { +) error{OutOfMemory}!void { const patch_size = dmp.match_max_bits; const patch_margin = dmp.patch_margin; const max_patch_len = patch_size - patch_margin; @@ -2811,7 +2817,7 @@ fn patchAddPadding( dmp: DiffMatchPatch, allocator: Allocator, patches: *PatchList, -) ![]const u8 { +) error{OutOfMemory}![]const u8 { if (patches.items.len == 0) return ""; const pad_len = dmp.patch_margin; var paddingcodes = try std.ArrayList(u8).initCapacity(allocator, pad_len); @@ -2899,7 +2905,7 @@ fn patchAddPadding( /// Given an array of patches, return another array that is identical. /// @param patches Array of Patch objects. /// @return Array of Patch objects. -fn patchListClone(allocator: Allocator, patches: *PatchList) !PatchList { +fn patchListClone(allocator: Allocator, patches: *PatchList) error{OutOfMemory}!PatchList { var new_patches = PatchList{}; errdefer deinitPatchList(allocator, &new_patches); try new_patches.ensureTotalCapacity(allocator, patches.items.len); @@ -2912,7 +2918,7 @@ fn patchListClone(allocator: Allocator, patches: *PatchList) !PatchList { /// Take a list of patches and return a textual representation. /// @param patches List of Patch objects. /// @return Text representation of patches. -pub fn patchToText(allocator: Allocator, patches: PatchList) ![]const u8 { +pub fn patchToText(allocator: Allocator, patches: PatchList) error{OutOfMemory}![]const u8 { var text_array = std.ArrayList(u8).init(allocator); defer text_array.deinit(); const writer = text_array.writer(); @@ -2932,7 +2938,7 @@ pub fn writePatch(writer: anytype, patches: PatchList) !void { /// @param textline Text representation of patches. /// @return List of Patch objects. /// @throws ArgumentException If invalid input. -pub fn patchFromText(allocator: Allocator, text: []const u8) !PatchList { +pub fn patchFromText(allocator: Allocator, text: []const u8) DiffError!PatchList { if (text.len == 0) return PatchList{}; var patches = PatchList{}; errdefer deinitPatchList(allocator, &patches); @@ -2947,7 +2953,7 @@ pub fn patchFromText(allocator: Allocator, text: []const u8) !PatchList { return patches; } -fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patch } { +fn patchFromHeader(allocator: Allocator, text: []const u8) DiffError!struct { usize, Patch } { var patch = Patch{ .diffs = DiffList{} }; errdefer patch.deinit(allocator); var cursor: usize = undefined; @@ -3072,7 +3078,7 @@ fn patchFromHeader(allocator: Allocator, text: []const u8) !struct { usize, Patc } /// Decode our URI-esque escaping -fn decodeUri(allocator: Allocator, line: []const u8) ![]const u8 { +fn decodeUri(allocator: Allocator, line: []const u8) DiffError![]const u8 { if (std.mem.indexOf(u8, line, "%")) |first| { // Text to decode. // Result will always be shorter than line: From b8d5661a1e39696c3d79fb30bd7a188fc7477be5 Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 5 Aug 2024 12:24:41 -0400 Subject: [PATCH 174/176] Panic? At the backout! --- DiffMatchPatch.zig | 4 ---- 1 file changed, 4 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 4f99198..423ec9e 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -1749,9 +1749,6 @@ fn diffCommonOverlap(text1_in: []const u8, text2_in: []const u8) usize { // trigger this code path. // XXX Remove this before merge if it can't be triggered. if (is_follow(text2[best_idx])) { - if (true) { - @panic("Your assumption regarding diffCommonOverlap is invalid!"); - } // back out return fixSplitBackward(text2, best_idx); } @@ -2090,7 +2087,6 @@ fn matchBitap( bin_max = bin_mid; var start: usize = @intCast(@max(1, i_loc - bin_mid + 1)); const finish: usize = @intCast(@min(i_loc + bin_mid, i_textlen) + i_patlen); - // No errors below this point, so no errdefer either: var rd: []usize = try allocator.alloc(usize, finish + 2); errdefer allocator.free(rd); const dshift: u6 = @intCast(d); From c87a288cfdb4c08480b2c7409e3fe06e0345252b Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 5 Aug 2024 19:37:54 -0400 Subject: [PATCH 175/176] Fix stack overflow in diffBisectSplit There's some odd false sharing happening which is codeunit related, but at least it will no longer aggressively blow stack. --- DiffMatchPatch.zig | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index 423ec9e..e44a028 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -788,6 +788,46 @@ fn diffBisectSplit( const text1b = text1[x1..]; const text2b = text2[y1..]; + if (text1a.len == 0 and text2a.len == 0) { + var diffs = DiffList{}; + errdefer deinitDiffList(allocator, &diffs); + try diffs.ensureUnusedCapacity(allocator, 2); + diffs.appendAssumeCapacity(Diff.init( + .delete, + try allocator.dupe( + u8, + text2b, + ), + )); + diffs.appendAssumeCapacity(Diff.init( + .insert, + try allocator.dupe( + u8, + text2b, + ), + )); + return diffs; + } else if (text1b.len == 0 and text2b.len == 0) { + var diffs = DiffList{}; + errdefer deinitDiffList(allocator, &diffs); + try diffs.ensureUnusedCapacity(allocator, 2); + diffs.appendAssumeCapacity(Diff.init( + .delete, + try allocator.dupe( + u8, + text2a, + ), + )); + diffs.appendAssumeCapacity(Diff.init( + .insert, + try allocator.dupe( + u8, + text2a, + ), + )); + return diffs; + } + // Compute both diffs serially. var diffs = try dmp.diffInternal(allocator, text1a, text2a, false, deadline); errdefer deinitDiffList(allocator, &diffs); @@ -4590,6 +4630,32 @@ test "Unicode diffs" { }, ); } + { + const before = "red blue green yellow"; + const after = "red♦︎ blue♦︎∅ green♦︎∅♦︎ yellow"; + var diffs = try dmp.diff(allocator, before, after, false); + defer deinitDiffList(allocator, &diffs); + } +} + +test "Workshop" { + const allocator = std.testing.allocator; + var dmp = DiffMatchPatch{}; + dmp.diff_timeout = 0; + { + const before = "red blue green yellow"; + const after = "red♦︎ blue♦︎∅ green♦︎∅♦︎ yellow"; + var diffs = try dmp.diff(allocator, before, after, false); + defer deinitDiffList(allocator, &diffs); + for (diffs.items) |a_diff| { + std.debug.print("{}\n", .{a_diff}); + std.debug.print(" {any}\n", .{a_diff.text}); + } + const before_2 = try diffBeforeText(allocator, diffs); + std.debug.print("{s}\n", .{before_2}); + defer allocator.free(before_2); + try testing.expectEqualStrings(before, before_2); + } } test "Diff format" { From 0249d68a6da5cb594e5df5e71f31cee48d98045f Mon Sep 17 00:00:00 2001 From: Sam Atman Date: Mon, 5 Aug 2024 20:26:54 -0400 Subject: [PATCH 176/176] Bugfix for the bugfix --- DiffMatchPatch.zig | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/DiffMatchPatch.zig b/DiffMatchPatch.zig index e44a028..8c4fb8f 100644 --- a/DiffMatchPatch.zig +++ b/DiffMatchPatch.zig @@ -796,7 +796,7 @@ fn diffBisectSplit( .delete, try allocator.dupe( u8, - text2b, + text1b, ), )); diffs.appendAssumeCapacity(Diff.init( @@ -815,7 +815,7 @@ fn diffBisectSplit( .delete, try allocator.dupe( u8, - text2a, + text2b, ), )); diffs.appendAssumeCapacity(Diff.init( @@ -4635,26 +4635,12 @@ test "Unicode diffs" { const after = "red♦︎ blue♦︎∅ green♦︎∅♦︎ yellow"; var diffs = try dmp.diff(allocator, before, after, false); defer deinitDiffList(allocator, &diffs); - } -} - -test "Workshop" { - const allocator = std.testing.allocator; - var dmp = DiffMatchPatch{}; - dmp.diff_timeout = 0; - { - const before = "red blue green yellow"; - const after = "red♦︎ blue♦︎∅ green♦︎∅♦︎ yellow"; - var diffs = try dmp.diff(allocator, before, after, false); - defer deinitDiffList(allocator, &diffs); - for (diffs.items) |a_diff| { - std.debug.print("{}\n", .{a_diff}); - std.debug.print(" {any}\n", .{a_diff.text}); - } const before_2 = try diffBeforeText(allocator, diffs); - std.debug.print("{s}\n", .{before_2}); defer allocator.free(before_2); try testing.expectEqualStrings(before, before_2); + const after_2 = try diffAfterText(allocator, diffs); + defer allocator.free(after_2); + try testing.expectEqualStrings(after, after_2); } }