support block comment

Saigut · Nov 10, 2017 · 1399ca2 · 1399ca2
1 parent 8da0852
commit 1399ca2
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 18 deletions.
diff --git a/src/schemely/highlighter/SchemeCommenter.java b/src/schemely/highlighter/SchemeCommenter.java
@@ -39,7 +39,7 @@ public String getCommentedBlockCommentSuffix()
   @Nullable
   public IElementType getLineCommentTokenType()
   {
-    return COMMENT;
+    return LINE_COMMENT;
   }
 
   @Nullable

diff --git a/src/schemely/highlighter/SchemeSyntaxHighlighter.java b/src/schemely/highlighter/SchemeSyntaxHighlighter.java
@@ -1,7 +1,6 @@
 package schemely.highlighter;
 
 import com.intellij.lexer.Lexer;
-import com.intellij.openapi.editor.DefaultLanguageHighlighterColors;
 import com.intellij.openapi.editor.HighlighterColors;
 import com.intellij.openapi.editor.SyntaxHighlighterColors;
 import com.intellij.openapi.editor.colors.TextAttributesKey;
@@ -141,7 +140,7 @@ public TextAttributesKey[] getTokenHighlights(IElementType tokenType)
 
   static
   {
-    newFillMap(ATTRIBUTES, LINE_COMMENT_KEYS, Tokens.COMMENT);
+    newFillMap(ATTRIBUTES, LINE_COMMENT_KEYS, Tokens.LINE_COMMENT);
     newFillMap(ATTRIBUTES, BLOCK_COMMENT_KEYS, Tokens.BLOCK_COMMENT);
     newFillMap(ATTRIBUTES, NUMBER_KEYS, Tokens.NUMBER_LITERAL);
     newFillMap(ATTRIBUTES, STRING_KEYS, Tokens.STRING_LITERAL);

diff --git a/src/schemely/lexer/SchemeLexer.java b/src/schemely/lexer/SchemeLexer.java
@@ -8,7 +8,6 @@
 import org.jparsec.pattern.Pattern;
 import org.jparsec.pattern.Patterns;
 
-import static org.jparsec.Scanners.lineComment;
 
 /**
  * @author Colin Fleming
@@ -31,7 +30,8 @@ public class SchemeLexer extends LexerBase
 
 
   enum Tag {
-    S_COMMENT,
+    S_LINE_COMMENT,
+    S_BLOCK_COMMENT,
     S_WHITE_SPACE,
     S_OPERATOR,
     S_NUMBER,
@@ -43,13 +43,33 @@ enum Tag {
     S_KEYWORD
   }
 
+  /**
+   * Helpers
+   */
+  private static Pattern notChar2(final char c1, final char c2) {
+    return new Pattern() {
+      @Override public int match(CharSequence src, int begin, int end) {
+        if (begin == end - 1) return 1;
+        if (begin >= end) return MISMATCH;
+        if (src.charAt(begin) == c1 && src.charAt(begin + 1) == c2) return Pattern.MISMATCH;
+        return 1;
+      }
+    };
+  }
+
   /**
    * Elements
    */
 
   // Blanks and Conments
-  Parser<?> s_line_comment = lineComment(";").source()
-          .map((a) -> (org.jparsec.Tokens.fragment(";", Tag.S_COMMENT)));
+  Parser<?> s_line_comment = Scanners.lineComment(";").source()
+          .map((a) -> (org.jparsec.Tokens.fragment(";", Tag.S_LINE_COMMENT)));
+  Parser<?> s_block_commented =
+          notChar2('|', '#').many().toScanner("commented block");
+  Parser<?> s_block_comment = Parsers.sequence(Scanners.string("#|"), s_block_commented, Scanners.string("|#")).source()
+          .map((a) -> (org.jparsec.Tokens.fragment(";", Tag.S_BLOCK_COMMENT)));
+  Parser<?> s_comment = Parsers.or(s_line_comment, s_block_comment);
+
   Parser<?> s_whitespace = Scanners.WHITESPACES
           .map((a) -> (org.jparsec.Tokens.fragment("WHITE_SPACE", Tag.S_WHITE_SPACE)));
 
@@ -99,7 +119,7 @@ enum Tag {
 
 
   // Bad char
-  Parser<?> s_element = Parsers.or(s_whitespace, s_line_comment,
+  Parser<?> s_element = Parsers.or(s_whitespace, s_comment,
           s_operators, s_numbers, s_keywords, s_literals);
 //  Parser<?> s_bad_element = s_element.not().source()
 //          .map((a) -> (org.jparsec.Tokens.fragment(a, Tag.S_BAD_ELEMENT)));
@@ -185,8 +205,12 @@ public void advance()
 
     switch ((Tag)token_frag.tag()) {
 
-      case S_COMMENT:
-        type = Tokens.COMMENT;
+      case S_LINE_COMMENT:
+        type = Tokens.LINE_COMMENT;
+        break;
+
+      case S_BLOCK_COMMENT:
+        type = Tokens.BLOCK_COMMENT;
         break;
 
       case S_WHITE_SPACE:
@@ -354,7 +378,7 @@ private void readSingleLineComment()
       cursor++;
       next = peek();
     }
-    type = Tokens.COMMENT;
+    type = Tokens.LINE_COMMENT;
   }
 
   private void readMultiLineComment()

diff --git a/src/schemely/lexer/Tokens.java b/src/schemely/lexer/Tokens.java
@@ -25,10 +25,10 @@ public interface Tokens
   IElementType COMMA_AT = new SchemeElementType(",@");
 
   // Comments
-  IElementType COMMENT = new SchemeElementType("comment");
+  IElementType LINE_COMMENT = new SchemeElementType("line comment");
   IElementType BLOCK_COMMENT = new SchemeElementType("block comment");
 
-  TokenSet COMMENTS = TokenSet.create(COMMENT, BLOCK_COMMENT);
+  TokenSet COMMENTS = TokenSet.create(LINE_COMMENT, BLOCK_COMMENT);
 
   // Literals
   IElementType STRING_LITERAL = new SchemeElementType("string literal");

diff --git a/test/scheme/lexer/LexerTest.java b/test/scheme/lexer/LexerTest.java
@@ -31,11 +31,11 @@ private static Object[][] lexerTestCases()
                               testCase(",", COMMA),
                               testCase(",@", COMMA_AT),
 
-                              testCase(";", COMMENT),
-                              testCase("; comment", COMMENT),
-                              testCase("; comment\n", COMMENT, WHITESPACE),
-                              testCase("; comment\r\n", COMMENT, WHITESPACE),
-                              testCase("; comment\r", COMMENT, WHITESPACE),
+                              testCase(";", LINE_COMMENT),
+                              testCase("; comment", LINE_COMMENT),
+                              testCase("; comment\n", LINE_COMMENT, WHITESPACE),
+                              testCase("; comment\r\n", LINE_COMMENT, WHITESPACE),
+                              testCase("; comment\r", LINE_COMMENT, WHITESPACE),
 
                               testCase("#||#", BLOCK_COMMENT),
                               testCase("#| |#", BLOCK_COMMENT),