Restore \n for all input in runtime tests, add extra LexerExec tests (LineSeparatorLf, LineSeparatorCrLf)

Author: KvanTTT

.gitattributes

@@ -1,4 +1,2 @@
 # This rule applies to all files which don't match another line below
-* text=auto
-LineSeparator_LF.txt eol=lf
-LineSeparator_CRLF.txt eol=crlf
+* text=auto

runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/LineSeparator_CRLF.txt

@@ -47,7 +47,6 @@ public abstract class BaseRuntimeTest {
 	final static Set<String> sections = new HashSet<>(Arrays.asList(
 		"notes", "type", "grammar", "slaveGrammar", "start", "input", "output", "errors", "flags", "skip"
 	));
-	final static Pattern linePattern = Pattern.compile("([^\\r\\n]*)(\r?\n)");
 
 	@BeforeClass
 	public static void startHeartbeatToAvoidTimeout() {
@@ -367,6 +366,12 @@ public static RuntimeTestDescriptor[] getRuntimeTestDescriptors(String group, St
 				System.err.println("Can't read descriptor file "+fname);
 			}
 		}
+
+		if (group.equals("LexerExec")) {
+			descriptors.add(ExtraTests.getLineSeparatorLfTest(targetName));
+			descriptors.add(ExtraTests.getLineSeparatorCrLfTest(targetName));
+		}
+
 		return descriptors.toArray(new RuntimeTestDescriptor[0]);
 	}
 
@@ -424,13 +429,9 @@ public static UniversalRuntimeTestDescriptor readDescriptor(String dtext)
 		StringBuilder currentValue = new StringBuilder();
 
 		List<Pair<String, String>> pairs = new ArrayList<>();
+		String[] lines = dtext.split("\r?\n");
 
-		Matcher matcher = linePattern.matcher(dtext);
-		boolean preserveSeparator = false;
-		int lastEnd = 0;
-		while (matcher.find()) {
-			String line = matcher.group(1);
-			lastEnd = matcher.end();
+		for (String line : lines) {
 			boolean newSection = false;
 			String sectionName = null;
 			if (line.startsWith("[") && line.length() > 2) {
@@ -439,19 +440,17 @@ public static UniversalRuntimeTestDescriptor readDescriptor(String dtext)
 			}
 
 			if (newSection) {
-				if (currentField!=null) {
+				if (currentField != null) {
 					pairs.add(new Pair<>(currentField, currentValue.toString()));
 				}
-				preserveSeparator = sectionName.equals("input");
 				currentField = sectionName;
 				currentValue.setLength(0);
 			}
 			else {
 				currentValue.append(line);
-				currentValue.append(preserveSeparator ? matcher.group(2) : "\n");
+				currentValue.append("\n");
 			}
 		}
-		currentValue.append(dtext.substring(lastEnd));
 		pairs.add(new Pair<>(currentField, currentValue.toString()));
 
 		UniversalRuntimeTestDescriptor d = new UniversalRuntimeTestDescriptor();

runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/LineSeparator_LF.txt

@@ -0,0 +1,41 @@
+package org.antlr.v4.test.runtime;
+
+public class ExtraTests {
+	static RuntimeTestDescriptor getLineSeparatorLfTest(String targetName) {
+		UniversalRuntimeTestDescriptor result = new UniversalRuntimeTestDescriptor();
+		result.name = "LineSeparatorLf";
+		result.targetName = targetName;
+		result.testType = "Lexer";
+		result.grammar = "lexer grammar L;\n" +
+				"T: ~'\\n'+;\n" +
+				"SEPARATOR: '\\n';";
+		result.grammarName = "L";
+		result.input = "1\n2\n3";
+		result.output = "[@0,0:0='1',<1>,1:0]\n" +
+				"[@1,1:1='\\n',<2>,1:1]\n" +
+				"[@2,2:2='2',<1>,2:0]\n" +
+				"[@3,3:3='\\n',<2>,2:1]\n" +
+				"[@4,4:4='3',<1>,3:0]\n" +
+				"[@5,5:4='<EOF>',<-1>,3:1]\n";
+		return result;
+	}
+
+	static RuntimeTestDescriptor getLineSeparatorCrLfTest(String targetName) {
+		UniversalRuntimeTestDescriptor result = new UniversalRuntimeTestDescriptor();
+		result.name = "LineSeparatorCrLf";
+		result.targetName = targetName;
+		result.testType = "Lexer";
+		result.grammar = "lexer grammar L;\n" +
+				"T: ~'\\r'+;\n" +
+				"SEPARATOR: '\\r\\n';";
+		result.grammarName = "L";
+		result.input = "1\r\n2\r\n3";
+		result.output = "[@0,0:0='1',<1>,1:0]\n" +
+				"[@1,1:2='\\r\\n',<2>,1:1]\n" +
+				"[@2,3:3='2',<1>,2:0]\n" +
+				"[@3,4:5='\\r\\n',<2>,2:1]\n" +
+				"[@4,6:6='3',<1>,3:0]\n" +
+				"[@5,7:6='<EOF>',<-1>,3:1]\n";
+		return result;
+	}
+}