Support Windows-1252 and make UTF-16 detection stricter

Fixes #797

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ### Parsing
 
+File detection now supports Windows-1252 encoded test (an extension of
+ISO-8859-1), and is stricter about UTF-16 detection.
+
 Updated to the latest tree-sitter parser for Make and YAML.
 
 ## 0.62 (released 20th December 2024)

Cargo.toml

@@ -97,6 +97,7 @@ tree-sitter-lua = "0.2.0"
 tree-sitter-xml = "0.7.0"
 tree-sitter-make = "1.1.1"
 tree-sitter-yaml = "0.7.0"
+encoding_rs = "0.8.35"
 
 [dev-dependencies]
 # assert_cmd 2.0.10 requires predicates 3.

sample_files/compare.expected

@@ -298,6 +298,9 @@ ca98b4d14fc21e0f04cf24aeb3d2526c  -
 sample_files/whitespace_1.tsx sample_files/whitespace_2.tsx
 ac8b1a89ac26333f2d4e9433b2ca3958  -
 
+sample_files/windows_1251_1.txt sample_files/windows_2251_1.txt
+d41d8cd98f00b204e9800998ecf8427e  -
+
 sample_files/xml_1.xml sample_files/xml_2.xml
 e629cbd2e721fd249c7ce1626f17e953  -
 

sample_files/windows_1251_1.txt

@@ -0,0 +1 @@
+Mu� k�nnen: l�st mu� da� Hei�t l�scht f�hren f�r mu� �hnlich

sample_files/windows_1251_2.txt

@@ -0,0 +1 @@
+Mu� k�nnen: l�st mu� da� Hei�t l�scht f�hren f�r mu� �hmlich

src/files.rs

@@ -215,14 +215,21 @@ pub(crate) fn guess_content(bytes: &[u8]) -> ProbableFileKind {
         .take(5000)
         .filter(|c| *c == std::char::REPLACEMENT_CHARACTER || *c == '\0')
         .count();
-    if num_utf16_invalid <= 5 {
+    if num_utf16_invalid <= 1 {
         info!(
             "Input file is mostly valid UTF-16 (invalid characters: {})",
             num_utf16_invalid
         );
         return ProbableFileKind::Text(utf16_string);
     }
 
+    // If the input bytes are valid Windows-1252 (an extension of
+    // ISO-8859-1 aka Latin 1), treat them as such.
+    let (latin1_str, _encoding, saw_malformed) = encoding_rs::WINDOWS_1252.decode(bytes);
+    if !saw_malformed {
+        return ProbableFileKind::Text(latin1_str.to_string());
+    }
+
     ProbableFileKind::Binary
 }