Change to ES6, made parser async, add async helpers (#294)

* initial commit

* naive touched up

* more async rework

* more async work

* missed one

* either done or next is lexing

* more fixes

* another package/tsconfig pass

* initial commit

* changing package version

* remove duplicate fn

* another async pass

* no-floating-promises

.eslintrc.js

@@ -14,6 +14,7 @@ module.exports = {
         "@typescript-eslint/consistent-type-assertions": ["warn", { assertionStyle: "as" }],
         "@typescript-eslint/explicit-function-return-type": "error",
         "@typescript-eslint/no-inferrable-types": "off",
+        "@typescript-eslint/no-floating-promises": "error",
         "@typescript-eslint/no-namespace": "error",
         "@typescript-eslint/no-unused-vars": ["error", { argsIgnorePattern: "^_" }],
         "@typescript-eslint/prefer-namespace-keyword": "error",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@microsoft/powerquery-parser",
-    "version": "0.5.23",
+    "version": "0.6.1",
     "description": "A parser for the Power Query/M formula language.",
     "author": "Microsoft",
     "license": "MIT",

src/example.ts

@@ -2,17 +2,18 @@
 // Licensed under the MIT license.
 
 /* eslint-disable @typescript-eslint/ban-ts-comment */
+/* eslint-disable @typescript-eslint/no-floating-promises */
 /* eslint-disable @typescript-eslint/no-unused-vars */
 /* eslint-disable no-unused-vars */
 
 import { Assert, DefaultSettings, Lexer, ResultUtils, Task, TaskUtils } from ".";
 
 parseText(`let x = 1 in try x otherwise 2`);
 
-function parseText(text: string): void {
+async function parseText(text: string): Promise<void> {
     // Try lexing and parsing the argument which returns a Result object.
     // A Result<T, E> is the union (Ok<T> | Error<E>).
-    const task: Task.TriedLexParseTask = TaskUtils.tryLexParse(DefaultSettings, text);
+    const task: Task.TriedLexParseTask = await TaskUtils.tryLexParse(DefaultSettings, text);
 
     // If it was a success then dump the abstract syntax tree (AST) as verbose JSON to console.
     if (TaskUtils.isParseStageOk(task)) {

src/powerquery-parser/common/arrayUtils.ts

@@ -63,6 +63,15 @@ export function assertNonZeroLength<T>(
     );
 }
 
+export async function asyncMap<T, U>(
+    collection: ReadonlyArray<T>,
+    mapFn: (value: T) => Promise<U>,
+): Promise<ReadonlyArray<U>> {
+    const tasks: ReadonlyArray<Promise<U>> = collection.map(mapFn);
+
+    return await Promise.all(tasks);
+}
+
 export function concatUnique<T>(left: ReadonlyArray<T>, right: ReadonlyArray<T>): ReadonlyArray<T> {
     const partial: T[] = [...left];
 

src/powerquery-parser/common/result/resultUtils.ts

@@ -35,3 +35,16 @@ export function ensureResult<T>(locale: string, callbackFn: () => T): Result<T,
         return boxError(CommonError.ensureCommonError(locale, error));
     }
 }
+
+export async function ensureAsyncResult<T>(
+    locale: string,
+    callbackFn: () => Promise<T>,
+): Promise<Result<T, CommonError.CommonError>> {
+    try {
+        return boxOk(await callbackFn());
+    } catch (error) {
+        Assert.isInstanceofError(error);
+
+        return boxError(CommonError.ensureCommonError(locale, error));
+    }
+}

src/powerquery-parser/parser/disambiguation/disambiguationUtils.ts

@@ -20,11 +20,11 @@ import { ParseError } from "..";
 // For each given parse function it'll create a deep copy of the state then parse with the function.
 // Mutates the given state to whatever parse state which matched the most amount of tokens.
 // Ties are resolved in the order of the given parse functions.
-export function readAmbiguous<T extends Ast.TNode>(
+export async function readAmbiguous<T extends Ast.TNode>(
     state: ParseState,
     parser: Parser,
-    parseFns: ReadonlyArray<(state: ParseState, parser: Parser) => T>,
-): AmbiguousParse<T> {
+    parseFns: ReadonlyArray<(state: ParseState, parser: Parser) => Promise<T>>,
+): Promise<AmbiguousParse<T>> {
     const trace: Trace = state.traceManager.entry(DisambiguationTraceConstant.Disambiguation, readAmbiguous.name, {
         [TraceConstant.Length]: parseFns.length,
     });
@@ -34,13 +34,15 @@ export function readAmbiguous<T extends Ast.TNode>(
     let maybeBestMatch: AmbiguousParse<T> | undefined = undefined;
 
     for (const parseFn of parseFns) {
-        const variantState: ParseState = parser.copyState(state);
+        // eslint-disable-next-line no-await-in-loop
+        const variantState: ParseState = await parser.copyState(state);
 
         let maybeNode: T | undefined;
         let variantResult: Result<T, ParseError.ParseError>;
 
         try {
-            maybeNode = parseFn(variantState, parser);
+            // eslint-disable-next-line no-await-in-loop
+            maybeNode = await parseFn(variantState, parser);
             variantResult = ResultUtils.boxOk(maybeNode);
         } catch (error) {
             if (!ParseError.isTInnerParseError(error)) {
@@ -65,11 +67,11 @@ export function readAmbiguous<T extends Ast.TNode>(
 }
 
 // Peeks at the token stream and either performs an explicit read or an ambiguous read.
-export function readAmbiguousBracket(
+export async function readAmbiguousBracket(
     state: ParseState,
     parser: Parser,
     allowedVariants: ReadonlyArray<BracketDisambiguation>,
-): TAmbiguousBracketNode {
+): Promise<TAmbiguousBracketNode> {
     const trace: Trace = state.traceManager.entry(
         DisambiguationTraceConstant.Disambiguation,
         readAmbiguousBracket.name,
@@ -86,15 +88,15 @@ export function readAmbiguousBracket(
 
         switch (disambiguation) {
             case BracketDisambiguation.FieldProjection:
-                ambiguousBracket = parser.readFieldProjection(state, parser);
+                ambiguousBracket = await parser.readFieldProjection(state, parser);
                 break;
 
             case BracketDisambiguation.FieldSelection:
-                ambiguousBracket = parser.readFieldSelection(state, parser);
+                ambiguousBracket = await parser.readFieldSelection(state, parser);
                 break;
 
             case BracketDisambiguation.RecordExpression:
-                ambiguousBracket = parser.readRecordExpression(state, parser);
+                ambiguousBracket = await parser.readRecordExpression(state, parser);
                 break;
 
             default:
@@ -110,7 +112,7 @@ export function readAmbiguousBracket(
                 throw ParseStateUtils.unterminatedBracketError(state);
 
             case DismabiguationBehavior.Thorough:
-                ambiguousBracket = thoroughReadAmbiguousBracket(state, parser, allowedVariants);
+                ambiguousBracket = await thoroughReadAmbiguousBracket(state, parser, allowedVariants);
                 break;
 
             default:
@@ -124,14 +126,17 @@ export function readAmbiguousBracket(
 }
 
 // Peeks at the token stream and either performs an explicit read or an ambiguous read.
-export function readAmbiguousParenthesis(state: ParseState, parser: Parser): TAmbiguousParenthesisNode {
+export async function readAmbiguousParenthesis(state: ParseState, parser: Parser): Promise<TAmbiguousParenthesisNode> {
     const trace: Trace = state.traceManager.entry(
         DisambiguationTraceConstant.Disambiguation,
         readAmbiguousParenthesis.name,
     );
 
     // We might be able to peek at tokens to disambiguate what parenthesized expression is next.
-    const maybeDisambiguation: ParenthesisDisambiguation | undefined = maybeDisambiguateParenthesis(state, parser);
+    const maybeDisambiguation: ParenthesisDisambiguation | undefined = await maybeDisambiguateParenthesis(
+        state,
+        parser,
+    );
 
     // Peeking gave us a concrete answer as to what's next.
     let ambiguousParenthesis: TAmbiguousParenthesisNode;
@@ -141,11 +146,11 @@ export function readAmbiguousParenthesis(state: ParseState, parser: Parser): TAm
 
         switch (disambiguation) {
             case ParenthesisDisambiguation.FunctionExpression:
-                ambiguousParenthesis = parser.readFunctionExpression(state, parser);
+                ambiguousParenthesis = await parser.readFunctionExpression(state, parser);
                 break;
 
             case ParenthesisDisambiguation.ParenthesizedExpression:
-                ambiguousParenthesis = readParenthesizedExpressionOrBinOpExpression(state, parser);
+                ambiguousParenthesis = await readParenthesizedExpressionOrBinOpExpression(state, parser);
                 break;
 
             default:
@@ -161,7 +166,7 @@ export function readAmbiguousParenthesis(state: ParseState, parser: Parser): TAm
                 throw ParseStateUtils.unterminatedParenthesesError(state);
 
             case DismabiguationBehavior.Thorough:
-                ambiguousParenthesis = thoroughReadAmbiguousParenthesis(state, parser);
+                ambiguousParenthesis = await thoroughReadAmbiguousParenthesis(state, parser);
                 break;
 
             default:
@@ -177,7 +182,10 @@ export function readAmbiguousParenthesis(state: ParseState, parser: Parser): TAm
 }
 
 // Peeks at tokens which might give a concrete disambiguation.
-export function maybeDisambiguateParenthesis(state: ParseState, parser: Parser): ParenthesisDisambiguation | undefined {
+export async function maybeDisambiguateParenthesis(
+    state: ParseState,
+    parser: Parser,
+): Promise<ParenthesisDisambiguation | undefined> {
     const trace: Trace = state.traceManager.entry(
         DisambiguationTraceConstant.Disambiguation,
         maybeDisambiguateParenthesis.name,
@@ -204,13 +212,16 @@ export function maybeDisambiguateParenthesis(state: ParseState, parser: Parser):
             // '(x as number) as number' could either be either case,
             // so we need to consume test if the trailing 'as number' is followed by a FatArrow.
             if (ParseStateUtils.isTokenKind(state, Token.TokenKind.KeywordAs, offsetTokenIndex + 1)) {
-                const checkpoint: ParseStateCheckpoint = parser.createCheckpoint(state);
+                // eslint-disable-next-line no-await-in-loop
+                const checkpoint: ParseStateCheckpoint = await parser.createCheckpoint(state);
                 unsafeMoveTo(state, offsetTokenIndex + 2);
 
                 try {
-                    parser.readNullablePrimitiveType(state, parser);
+                    // eslint-disable-next-line no-await-in-loop
+                    await parser.readNullablePrimitiveType(state, parser);
                 } catch {
-                    parser.restoreCheckpoint(state, checkpoint);
+                    // eslint-disable-next-line no-await-in-loop
+                    await parser.restoreCheckpoint(state, checkpoint);
 
                     if (ParseStateUtils.isOnTokenKind(state, Token.TokenKind.FatArrow)) {
                         return ParenthesisDisambiguation.FunctionExpression;
@@ -225,7 +236,8 @@ export function maybeDisambiguateParenthesis(state: ParseState, parser: Parser):
                     maybeDisambiguation = ParenthesisDisambiguation.ParenthesizedExpression;
                 }
 
-                parser.restoreCheckpoint(state, checkpoint);
+                // eslint-disable-next-line no-await-in-loop
+                await parser.restoreCheckpoint(state, checkpoint);
             } else if (ParseStateUtils.isTokenKind(state, Token.TokenKind.FatArrow, offsetTokenIndex + 1)) {
                 maybeDisambiguation = ParenthesisDisambiguation.FunctionExpression;
             } else {
@@ -304,17 +316,17 @@ const enum DisambiguationTraceConstant {
 // Copy the current state and attempt to read for each of the following:
 //  FieldProjection, FieldSelection, and RecordExpression.
 // Mutates the given state with the read attempt which matched the most tokens.
-function thoroughReadAmbiguousBracket(
+async function thoroughReadAmbiguousBracket(
     state: ParseState,
     parser: Parser,
     allowedVariants: ReadonlyArray<BracketDisambiguation>,
-): TAmbiguousBracketNode {
+): Promise<TAmbiguousBracketNode> {
     const trace: Trace = state.traceManager.entry(
         DisambiguationTraceConstant.Disambiguation,
         readAmbiguousBracket.name,
     );
 
-    const ambiguousBracket: TAmbiguousBracketNode = thoroughReadAmbiguous(
+    const ambiguousBracket: TAmbiguousBracketNode = await thoroughReadAmbiguous(
         state,
         parser,
         bracketDisambiguationParseFunctions(parser, allowedVariants),
@@ -328,18 +340,18 @@ function thoroughReadAmbiguousBracket(
 // Copy the current state and attempt to read for each of the following:
 //  FunctionExpression, ParenthesisExpression.
 // Mutates the given state with the read attempt which matched the most tokens.
-function thoroughReadAmbiguousParenthesis(state: ParseState, parser: Parser): TAmbiguousParenthesisNode {
-    return thoroughReadAmbiguous<TAmbiguousParenthesisNode>(state, parser, [
+async function thoroughReadAmbiguousParenthesis(state: ParseState, parser: Parser): Promise<TAmbiguousParenthesisNode> {
+    return await thoroughReadAmbiguous<TAmbiguousParenthesisNode>(state, parser, [
         parser.readFunctionExpression,
         readParenthesizedExpressionOrBinOpExpression,
     ]);
 }
 
-function thoroughReadAmbiguous<T extends TAmbiguousBracketNode | TAmbiguousParenthesisNode>(
+async function thoroughReadAmbiguous<T extends TAmbiguousBracketNode | TAmbiguousParenthesisNode>(
     state: ParseState,
     parser: Parser,
-    parseFns: ReadonlyArray<(state: ParseState, parser: Parser) => T>,
-): T {
+    parseFns: ReadonlyArray<(state: ParseState, parser: Parser) => Promise<T>>,
+): Promise<T> {
     const trace: Trace = state.traceManager.entry(
         DisambiguationTraceConstant.Disambiguation,
         thoroughReadAmbiguous.name,
@@ -348,9 +360,9 @@ function thoroughReadAmbiguous<T extends TAmbiguousBracketNode | TAmbiguousParen
         },
     );
 
-    const ambiguousParse: AmbiguousParse<T> = readAmbiguous(state, parser, parseFns);
+    const ambiguousParse: AmbiguousParse<T> = await readAmbiguous(state, parser, parseFns);
 
-    parser.applyState(state, ambiguousParse.parseState);
+    await parser.applyState(state, ambiguousParse.parseState);
 
     if (ResultUtils.isOk(ambiguousParse.result)) {
         trace.exit({
@@ -376,7 +388,7 @@ function thoroughReadAmbiguous<T extends TAmbiguousBracketNode | TAmbiguousParen
 function bracketDisambiguationParseFunctions(
     parser: Parser,
     allowedVariants: ReadonlyArray<BracketDisambiguation>,
-): ReadonlyArray<(state: ParseState, parser: Parser) => TAmbiguousBracketNode> {
+): ReadonlyArray<(state: ParseState, parser: Parser) => Promise<TAmbiguousBracketNode>> {
     return allowedVariants.map((bracketDisambiguation: BracketDisambiguation) => {
         switch (bracketDisambiguation) {
             case BracketDisambiguation.FieldProjection:
@@ -397,11 +409,11 @@ function bracketDisambiguationParseFunctions(
 // When the next token is an open parenthesis we can't directly read
 // a ParenthesisExpression as it may leave trailing tokens behind.
 // `(1) + 2`
-function readParenthesizedExpressionOrBinOpExpression(
+async function readParenthesizedExpressionOrBinOpExpression(
     state: ParseState,
     parser: Parser,
-): Ast.ParenthesizedExpression | Ast.TLogicalExpression {
-    const node: Ast.TNode = parser.readLogicalExpression(state, parser);
+): Promise<Ast.ParenthesizedExpression | Ast.TLogicalExpression> {
+    const node: Ast.TNode = await parser.readLogicalExpression(state, parser);
 
     const leftMostNode: Ast.TNode = NodeIdMapUtils.assertUnboxLeftMostLeaf(
         state.contextState.nodeIdMapCollection,

src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/leafSelectors.ts

@@ -58,7 +58,7 @@ export function maybeRightMostLeaf(
     nodeIdMapCollection: Collection,
     rootId: number,
     maybeCondition: ((node: Ast.TNode) => boolean) | undefined = undefined,
-): Ast.TNode | undefined {
+): Promise<Ast.TNode | undefined> {
     const astNodeById: AstNodeById = nodeIdMapCollection.astNodeById;
     let nodeIdsToExplore: number[] = [rootId];
     let maybeRightMost: Ast.TNode | undefined;
@@ -119,13 +119,13 @@ export function maybeRightMostLeaf(
         }
     }
 
-    return maybeRightMost;
+    return Promise.resolve(maybeRightMost);
 }
 
 export function maybeRightMostLeafWhere(
     nodeIdMapCollection: Collection,
     rootId: number,
     maybeCondition: ((node: Ast.TNode) => boolean) | undefined,
-): Ast.TNode | undefined {
+): Promise<Ast.TNode | undefined> {
     return maybeRightMostLeaf(nodeIdMapCollection, rootId, maybeCondition);
 }

src/powerquery-parser/parser/nodeIdMap/nodeIdMapUtils/nodeIdMapUtils.ts

@@ -69,7 +69,10 @@ export function hasParsedToken(nodeIdMapCollection: Collection, nodeId: number):
     return false;
 }
 
-export function xorNodeTokenRange(nodeIdMapCollection: Collection, xorNode: TXorNode): XorNodeTokenRange {
+export async function xorNodeTokenRange(
+    nodeIdMapCollection: Collection,
+    xorNode: TXorNode,
+): Promise<XorNodeTokenRange> {
     switch (xorNode.kind) {
         case XorNodeKind.Ast: {
             const tokenRange: Token.TokenRange = xorNode.node.tokenRange;
@@ -84,7 +87,10 @@ export function xorNodeTokenRange(nodeIdMapCollection: Collection, xorNode: TXor
             const contextNode: ParseContext.TNode = xorNode.node;
             let tokenIndexEnd: number;
 
-            const maybeRightMostChild: Ast.TNode | undefined = maybeRightMostLeaf(nodeIdMapCollection, xorNode.node.id);
+            const maybeRightMostChild: Ast.TNode | undefined = await maybeRightMostLeaf(
+                nodeIdMapCollection,
+                xorNode.node.id,
+            );
 
             if (maybeRightMostChild === undefined) {
                 tokenIndexEnd = contextNode.tokenIndexStart;

src/powerquery-parser/parser/parseState/parseStateUtils.ts

@@ -45,8 +45,8 @@ export function createState(lexerSnapshot: LexerSnapshot, maybeOverrides: Partia
 }
 
 // If you have a custom parser + parser state, then you'll have to create your own copyState/applyState functions.
-// See `benchmark.ts` for an example.
-export function applyState(state: ParseState, update: ParseState): void {
+// eslint-disable-next-line require-await
+export async function applyState(state: ParseState, update: ParseState): Promise<void> {
     state.tokenIndex = update.tokenIndex;
     state.maybeCurrentToken = update.maybeCurrentToken;
     state.maybeCurrentTokenKind = update.maybeCurrentTokenKind;
@@ -55,8 +55,8 @@ export function applyState(state: ParseState, update: ParseState): void {
 }
 
 // If you have a custom parser + parser state, then you'll have to create your own copyState/applyState functions.
-// See `benchmark.ts` for an example.
-export function copyState(state: ParseState): ParseState {
+// eslint-disable-next-line require-await
+export async function copyState(state: ParseState): Promise<ParseState> {
     return {
         ...state,
         contextState: ParseContextUtils.copyState(state.contextState),