diff --git a/src/vm/wren_compiler.c b/src/vm/wren_compiler.c index 001c94c36..31c0fd0f5 100644 --- a/src/vm/wren_compiler.c +++ b/src/vm/wren_compiler.c @@ -173,6 +173,9 @@ typedef struct // The 1-based line number of [currentChar]. int currentLine; + // The upcoming token. + Token next; + // The most recently lexed token. Token current; @@ -639,13 +642,13 @@ static bool matchChar(Parser* parser, char c) // range. static void makeToken(Parser* parser, TokenType type) { - parser->current.type = type; - parser->current.start = parser->tokenStart; - parser->current.length = (int)(parser->currentChar - parser->tokenStart); - parser->current.line = parser->currentLine; + parser->next.type = type; + parser->next.start = parser->tokenStart; + parser->next.length = (int)(parser->currentChar - parser->tokenStart); + parser->next.line = parser->currentLine; // Make line tokens appear on the line containing the "\n". - if (type == TOKEN_LINE) parser->current.line--; + if (type == TOKEN_LINE) parser->next.line--; } // If the current character is [c], then consumes it and makes a token of type @@ -719,17 +722,17 @@ static void makeNumber(Parser* parser, bool isHex) if (isHex) { - parser->current.value = NUM_VAL((double)strtoll(parser->tokenStart, NULL, 16)); + parser->next.value = NUM_VAL((double)strtoll(parser->tokenStart, NULL, 16)); } else { - parser->current.value = NUM_VAL(strtod(parser->tokenStart, NULL)); + parser->next.value = NUM_VAL(strtod(parser->tokenStart, NULL)); } if (errno == ERANGE) { lexError(parser, "Number literal was too large (%d).", sizeof(long int)); - parser->current.value = NUM_VAL(0); + parser->next.value = NUM_VAL(0); } // We don't check that the entire token is consumed after calling strtoll() @@ -921,21 +924,23 @@ static void readString(Parser* parser) } } - parser->current.value = wrenNewStringLength(parser->vm, + parser->next.value = wrenNewStringLength(parser->vm, (char*)string.data, string.count); wrenByteBufferClear(parser->vm, &string); makeToken(parser, type); } -// Lex the next token and store it in [parser.current]. +// Lex the next token and store it in [parser.next]. static void nextToken(Parser* parser) { parser->previous = parser->current; + parser->current = parser->next; // If we are out of tokens, don't try to tokenize any more. We *do* still // copy the TOKEN_EOF to previous so that code that expects it to be consumed // will still work. + if (parser->next.type == TOKEN_EOF) return; if (parser->current.type == TOKEN_EOF) return; while (peekChar(parser) != '\0') @@ -1094,8 +1099,8 @@ static void nextToken(Parser* parser) // even though the source code and console output are UTF-8. lexError(parser, "Invalid byte 0x%x.", (uint8_t)c); } - parser->current.type = TOKEN_ERROR; - parser->current.length = 0; + parser->next.type = TOKEN_ERROR; + parser->next.length = 0; } return; } @@ -1114,6 +1119,12 @@ static TokenType peek(Compiler* compiler) return compiler->parser->current.type; } +// Returns the type of the current token. +static TokenType peekNext(Compiler* compiler) +{ + return compiler->parser->next.type; +} + // Consumes the current token if its type is [expected]. Returns true if a // token was consumed. static bool match(Compiler* compiler, TokenType expected) @@ -1162,6 +1173,12 @@ static void consumeLine(Compiler* compiler, const char* errorMessage) ignoreNewlines(compiler); } +static void allowLineBeforeDot(Compiler* compiler) { + if (peek(compiler) == TOKEN_LINE && peekNext(compiler) == TOKEN_DOT) { + nextToken(compiler->parser); + } +} + // Variables and scopes -------------------------------------------------------- // Emits one single-byte argument. Returns its index. @@ -1949,6 +1966,7 @@ static void namedCall(Compiler* compiler, bool canAssign, Code instruction) else { methodCall(compiler, instruction, &signature); + allowLineBeforeDot(compiler); } } @@ -2145,6 +2163,8 @@ static void field(Compiler* compiler, bool canAssign) loadThis(compiler); emitByteArg(compiler, isLoad ? CODE_LOAD_FIELD : CODE_STORE_FIELD, field); } + + allowLineBeforeDot(compiler); } // Compiles a read or assignment to [variable]. @@ -2176,6 +2196,8 @@ static void bareName(Compiler* compiler, bool canAssign, Variable variable) // Emit the load instruction. loadVariable(compiler, variable); + + allowLineBeforeDot(compiler); } static void staticField(Compiler* compiler, bool canAssign) @@ -2360,6 +2382,8 @@ static void subscript(Compiler* compiler, bool canAssign) finishArgumentList(compiler, &signature); consume(compiler, TOKEN_RIGHT_BRACKET, "Expect ']' after arguments."); + allowLineBeforeDot(compiler); + if (canAssign && match(compiler, TOKEN_EQ)) { signature.type = SIG_SUBSCRIPT_SETTER; @@ -3488,19 +3512,21 @@ ObjFn* wrenCompile(WrenVM* vm, ObjModule* module, const char* source, parser.numParens = 0; // Zero-init the current token. This will get copied to previous when - // advance() is called below. - parser.current.type = TOKEN_ERROR; - parser.current.start = source; - parser.current.length = 0; - parser.current.line = 0; - parser.current.value = UNDEFINED_VAL; + // nextToken() is called below. + parser.next.type = TOKEN_ERROR; + parser.next.start = source; + parser.next.length = 0; + parser.next.line = 0; + parser.next.value = UNDEFINED_VAL; // Ignore leading newlines. parser.skipNewlines = true; parser.printErrors = printErrors; parser.hasError = false; - // Read the first token. + // Read the first token into next + nextToken(&parser); + // Copy next -> current nextToken(&parser); int numExistingVariables = module->variables.count; diff --git a/test/language/chained_newline.wren b/test/language/chained_newline.wren new file mode 100644 index 000000000..bd1aaf847 --- /dev/null +++ b/test/language/chained_newline.wren @@ -0,0 +1,151 @@ +class Test { + construct new() {} + test0() { + System.print("test0") + return this + } + test1() { + System.print("test1") + return this + } + test2() { + System.print("test2") + return this + } + + [index] { + System.print("testSubscript") + return this + } +} + +class Tester { + construct new() { + + var test = _test = Test.new() + + //test local access + + test. + test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 + + test + .test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 + + test + .test0() // expect: test0 + .test1(). // expect: test1 + test2() // expect: test2 + + test[0] // expect: testSubscript + .test0() // expect: test0 + + test[0]. // expect: testSubscript + test0() // expect: test0 + + //test field access + + _test. + test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 + + _test + .test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 + + _test + .test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 + + _test[0] // expect: testSubscript + .test0() // expect: test0 + + _test[0]. // expect: testSubscript + test0() // expect: test0 + + } + + getter { _test } + method() { _test } + +} //Tester + +//access via methods/getter + +var external = Tester.new() + +external.getter. + test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 + +external.getter + .test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 + +external.getter. + test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 + +external.getter[0]. // expect: testSubscript + test0() // expect: test0 + +external.getter[0] // expect: testSubscript + .test0() // expect: test0 + +external.method(). + test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 + +external.method() + .test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 + +external.method(). + test0() // expect: test0 + .test1(). // expect: test1 + test2() // expect: test2 + +external.method()[0]. // expect: testSubscript + test0() // expect: test0 + +external.method()[0] // expect: testSubscript + .test0() // expect: test0 + + +//regular access in module scope + +var other = Test.new() + +other. + test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 + +other + .test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 + +other + .test0(). // expect: test0 + test1() // expect: test1 + .test2() // expect: test2 + + +other[0] // expect: testSubscript + .test0() // expect: test0 + +other[0]. // expect: testSubscript + test0() // expect: test0