Make Traverse API async (#295)

* initial commit

* more impl, all tests pass

* alpha sort

* consistent async fn naming

.eslintrc.js

@@ -13,8 +13,8 @@ module.exports = {
         "@typescript-eslint/await-thenable": "error",
         "@typescript-eslint/consistent-type-assertions": ["warn", { assertionStyle: "as" }],
         "@typescript-eslint/explicit-function-return-type": "error",
-        "@typescript-eslint/no-inferrable-types": "off",
         "@typescript-eslint/no-floating-promises": "error",
+        "@typescript-eslint/no-inferrable-types": "off",
         "@typescript-eslint/no-namespace": "error",
         "@typescript-eslint/no-unused-vars": ["error", { argsIgnorePattern: "^_" }],
         "@typescript-eslint/prefer-namespace-keyword": "error",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@microsoft/powerquery-parser",
-    "version": "0.6.1",
+    "version": "0.6.2",
     "description": "A parser for the Power Query/M formula language.",
     "author": "Microsoft",
     "license": "MIT",

src/powerquery-parser/common/arrayUtils.ts

@@ -63,7 +63,7 @@ export function assertNonZeroLength<T>(
     );
 }
 
-export async function asyncMap<T, U>(
+export async function mapAsync<T, U>(
     collection: ReadonlyArray<T>,
     mapFn: (value: T) => Promise<U>,
 ): Promise<ReadonlyArray<U>> {

src/powerquery-parser/common/result/resultUtils.ts

@@ -36,7 +36,7 @@ export function ensureResult<T>(locale: string, callbackFn: () => T): Result<T,
     }
 }
 
-export async function ensureAsyncResult<T>(
+export async function ensureResultAsync<T>(
     locale: string,
     callbackFn: () => Promise<T>,
 ): Promise<Result<T, CommonError.CommonError>> {

src/powerquery-parser/common/traversal.ts

@@ -1,23 +1,19 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-import { Assert, CommonError, Result } from ".";
+import { ArrayUtils, Assert, CommonError, Result } from ".";
 import { NodeIdMap, NodeIdMapUtils, ParseContext, TXorNode, XorNodeKind, XorNodeUtils } from "../parser";
 import { Ast } from "../language";
 import { ResultUtils } from "./result";
+import { Settings } from "..";
+import { Trace } from "./trace";
 
 export type TriedTraverse<ResultType> = Result<ResultType, CommonError.CommonError>;
 
 export type TVisitNodeFn<State extends ITraversalState<ResultType>, ResultType, Node, Return> = (
     state: State,
     node: Node,
-) => Return;
-
-export type TVisitChildNodeFn<State extends ITraversalState<ResultType>, ResultType, Node, Return> = (
-    state: State,
-    parent: Node,
-    node: Node,
-) => Return;
+) => Promise<Return>;
 
 export type TEarlyExitFn<State extends ITraversalState<ResultType>, ResultType, Node> = TVisitNodeFn<
     State,
@@ -30,15 +26,14 @@ export type TExpandNodesFn<State extends ITraversalState<ResultType>, ResultType
     state: State,
     node: Node,
     collection: NodesById,
-) => ReadonlyArray<Node>;
+) => Promise<ReadonlyArray<Node>>;
 
 export const enum VisitNodeStrategy {
     BreadthFirst = "BreadthFirst",
     DepthFirst = "DepthFirst",
 }
 
-export interface ITraversalState<T> {
-    readonly locale: string;
+export interface ITraversalState<T> extends Pick<Settings, "locale" | "maybeCancellationToken" | "traceManager"> {
     result: T;
 }
 
@@ -51,7 +46,7 @@ export function tryTraverseAst<State extends ITraversalState<ResultType>, Result
     visitNodeFn: TVisitNodeFn<State, ResultType, Ast.TNode, void>,
     expandNodesFn: TExpandNodesFn<State, ResultType, Ast.TNode, NodeIdMap.Collection>,
     maybeEarlyExitFn: TEarlyExitFn<State, ResultType, Ast.TNode> | undefined,
-): TriedTraverse<ResultType> {
+): Promise<TriedTraverse<ResultType>> {
     return tryTraverse<State, ResultType, Ast.TNode, NodeIdMap.Collection>(
         state,
         nodeIdMapCollection,
@@ -72,7 +67,7 @@ export function tryTraverseXor<State extends ITraversalState<ResultType>, Result
     visitNodeFn: TVisitNodeFn<State, ResultType, TXorNode, void>,
     expandNodesFn: TExpandNodesFn<State, ResultType, TXorNode, NodeIdMap.Collection>,
     maybeEarlyExitFn: TEarlyExitFn<State, ResultType, TXorNode> | undefined,
-): TriedTraverse<ResultType> {
+): Promise<TriedTraverse<ResultType>> {
     return tryTraverse<State, ResultType, TXorNode, NodeIdMap.Collection>(
         state,
         nodeIdMapCollection,
@@ -92,9 +87,9 @@ export function tryTraverse<State extends ITraversalState<ResultType>, ResultTyp
     visitNodeFn: TVisitNodeFn<State, ResultType, Node, void>,
     expandNodesFn: TExpandNodesFn<State, ResultType, Node, NodesById>,
     maybeEarlyExitFn: TEarlyExitFn<State, ResultType, Node> | undefined,
-): TriedTraverse<ResultType> {
-    return ResultUtils.ensureResult(state.locale, () => {
-        traverseRecursion<State, ResultType, Node, NodesById>(
+): Promise<TriedTraverse<ResultType>> {
+    return ResultUtils.ensureResultAsync(state.locale, async () => {
+        await traverseRecursion<State, ResultType, Node, NodesById>(
             state,
             nodesById,
             root,
@@ -109,11 +104,12 @@ export function tryTraverse<State extends ITraversalState<ResultType>, ResultTyp
 }
 
 // a TExpandNodesFn usable by tryTraverseAst which visits all nodes.
-export function assertGetAllAstChildren<State extends ITraversalState<ResultType>, ResultType>(
+// eslint-disable-next-line require-await
+export async function assertGetAllAstChildren<State extends ITraversalState<ResultType>, ResultType>(
     _state: State,
     astNode: Ast.TNode,
     nodeIdMapCollection: NodeIdMap.Collection,
-): ReadonlyArray<Ast.TNode> {
+): Promise<ReadonlyArray<Ast.TNode>> {
     const maybeChildIds: ReadonlyArray<number> | undefined = nodeIdMapCollection.childIdsById.get(astNode.id);
 
     if (maybeChildIds) {
@@ -126,16 +122,23 @@ export function assertGetAllAstChildren<State extends ITraversalState<ResultType
 }
 
 // a TExpandNodesFn usable by tryTraverseXor which visits all nodes.
-export function assertGetAllXorChildren<State extends ITraversalState<ResultType>, ResultType>(
+// eslint-disable-next-line require-await
+export async function assertGetAllXorChildren<State extends ITraversalState<ResultType>, ResultType>(
     _state: State,
     xorNode: TXorNode,
     nodeIdMapCollection: NodeIdMap.Collection,
-): ReadonlyArray<TXorNode> {
+): Promise<ReadonlyArray<TXorNode>> {
     switch (xorNode.kind) {
         case XorNodeKind.Ast: {
             const astNode: Ast.TNode = xorNode.node;
 
-            return assertGetAllAstChildren(_state, astNode, nodeIdMapCollection).map(XorNodeUtils.boxAst);
+            const children: ReadonlyArray<Ast.TNode> = await assertGetAllAstChildren(
+                _state,
+                astNode,
+                nodeIdMapCollection,
+            );
+
+            return ArrayUtils.mapAsync(children, (value: Ast.TNode) => Promise.resolve(XorNodeUtils.boxAst(value)));
         }
 
         case XorNodeKind.Context: {
@@ -167,32 +170,38 @@ export function maybeExpandXorParent<T>(
     _state: T,
     xorNode: TXorNode,
     nodeIdMapCollection: NodeIdMap.Collection,
-): ReadonlyArray<TXorNode> {
+): Promise<ReadonlyArray<TXorNode>> {
     const maybeParent: TXorNode | undefined = NodeIdMapUtils.maybeParentXor(nodeIdMapCollection, xorNode.node.id);
 
-    return maybeParent !== undefined ? [maybeParent] : [];
+    return Promise.resolve(maybeParent !== undefined ? [maybeParent] : []);
 }
 
-function traverseRecursion<State extends ITraversalState<ResultType>, ResultType, Node, NodesById>(
+async function traverseRecursion<State extends ITraversalState<ResultType>, ResultType, Node, NodesById>(
     state: State,
     nodesById: NodesById,
     node: Node,
     strategy: VisitNodeStrategy,
     visitNodeFn: TVisitNodeFn<State, ResultType, Node, void>,
     expandNodesFn: TExpandNodesFn<State, ResultType, Node, NodesById>,
     maybeEarlyExitFn: TEarlyExitFn<State, ResultType, Node> | undefined,
-): void {
-    if (maybeEarlyExitFn && maybeEarlyExitFn(state, node)) {
+): Promise<void> {
+    const trace: Trace = state.traceManager.entry("Traversal", traverseRecursion.name);
+    state.maybeCancellationToken?.throwIfCancelled();
+
+    if (maybeEarlyExitFn && (await maybeEarlyExitFn(state, node))) {
         return;
     } else if (strategy === VisitNodeStrategy.BreadthFirst) {
-        visitNodeFn(state, node);
+        await visitNodeFn(state, node);
     }
 
-    for (const child of expandNodesFn(state, node, nodesById)) {
-        traverseRecursion(state, nodesById, child, strategy, visitNodeFn, expandNodesFn, maybeEarlyExitFn);
+    for (const child of await expandNodesFn(state, node, nodesById)) {
+        // eslint-disable-next-line no-await-in-loop
+        await traverseRecursion(state, nodesById, child, strategy, visitNodeFn, expandNodesFn, maybeEarlyExitFn);
     }
 
     if (strategy === VisitNodeStrategy.DepthFirst) {
-        visitNodeFn(state, node);
+        await visitNodeFn(state, node);
     }
+
+    trace.exit();
 }

src/test/libraryTest/parser/children.ts

@@ -29,7 +29,7 @@ function createActual(lexParseOk: Task.ParseTaskOk): ChildIdsByIdEntry[] {
 }
 
 describe("Parser.Children", () => {
-    it(`WIP () as number => 1`, async () => {
+    it(`() as number => 1`, async () => {
         const text: string = `() as number => 1`;
 
         const expected: ReadonlyArray<ChildIdsByIdEntry> = [

src/test/libraryTest/parser/idUtils.ts

@@ -8,6 +8,7 @@ import { Assert, Language, TaskUtils, Traverse } from "../../../powerquery-parse
 import { ChildIdsById, IdsByNodeKind, ParentIdById } from "../../../powerquery-parser/parser/nodeIdMap/nodeIdMap";
 import { DefaultSettings, Task } from "../../..";
 import { NodeIdMap, TXorNode, XorNodeUtils } from "../../../powerquery-parser/parser";
+import { NoOpTraceManager } from "../../../powerquery-parser/common/trace";
 
 type TraverseState = Traverse.ITraversalState<undefined> &
     Pick<NodeIdMap.Collection, "leafIds" | "idsByNodeKind"> & { astIds: number[]; contextIds: number[] };
@@ -48,7 +49,10 @@ function createSimplifiedParentIdById(parentIdById: ParentIdById): ReadonlyArray
     return [...parentIdById.entries()].sort();
 }
 
-function expectLinksMatch(triedLexParse: Task.TriedLexParseTask, expected: AbridgedNodeIdMapCollection): void {
+async function expectLinksMatch(
+    triedLexParse: Task.TriedLexParseTask,
+    expected: AbridgedNodeIdMapCollection,
+): Promise<void> {
     let nodeIdMapCollection: NodeIdMap.Collection;
     let xorNode: TXorNode;
 
@@ -78,9 +82,11 @@ function expectLinksMatch(triedLexParse: Task.TriedLexParseTask, expected: Abrid
         contextIds: [],
         leafIds: new Set<number>(),
         idsByNodeKind: new Map(),
+        maybeCancellationToken: undefined,
+        traceManager: new NoOpTraceManager(),
     };
 
-    const triedTraverse: Traverse.TriedTraverse<undefined> = Traverse.tryTraverseXor(
+    const triedTraverse: Traverse.TriedTraverse<undefined> = await Traverse.tryTraverseXor(
         traverseState,
         triedLexParse.nodeIdMapCollection,
         xorNode,
@@ -185,7 +191,8 @@ function assertTraverseMatchesState(traverseState: TraverseState, nodeIdMapColle
     );
 }
 
-function traverseVisitNode(state: TraverseState, xorNode: TXorNode): void {
+// eslint-disable-next-line require-await
+async function traverseVisitNode(state: TraverseState, xorNode: TXorNode): Promise<void> {
     if (XorNodeUtils.isAstXor(xorNode)) {
         state.astIds.push(xorNode.node.id);
 
@@ -219,7 +226,7 @@ describe("idUtils", () => {
         };
 
         const triedLexParse: Task.TriedLexParseTask = await TaskUtils.tryLexParse(DefaultSettings, text);
-        expectLinksMatch(triedLexParse, expected);
+        await expectLinksMatch(triedLexParse, expected);
     });
 
     it(`-1`, async () => {
@@ -247,7 +254,7 @@ describe("idUtils", () => {
         };
 
         const triedLexParse: Task.TriedLexParseTask = await TaskUtils.tryLexParse(DefaultSettings, text);
-        expectLinksMatch(triedLexParse, expected);
+        await expectLinksMatch(triedLexParse, expected);
     });
 
     it(`1 + 2`, async () => {
@@ -271,7 +278,7 @@ describe("idUtils", () => {
         };
 
         const triedLexParse: Task.TriedLexParseTask = await TaskUtils.tryLexParse(DefaultSettings, text);
-        expectLinksMatch(triedLexParse, expected);
+        await expectLinksMatch(triedLexParse, expected);
     });
 
     it(`foo()`, async () => {
@@ -307,6 +314,6 @@ describe("idUtils", () => {
         };
 
         const triedLexParse: Task.TriedLexParseTask = await TaskUtils.tryLexParse(DefaultSettings, text);
-        expectLinksMatch(triedLexParse, expected);
+        await expectLinksMatch(triedLexParse, expected);
     });
 });

src/test/libraryTest/parser/simple.ts

@@ -15,6 +15,7 @@ import {
     TaskUtils,
     Traverse,
 } from "../../..";
+import { NoOpTraceManager } from "../../../powerquery-parser/common/trace";
 import { TestAssertUtils } from "../../testUtils";
 
 type AbridgedNode = [Language.Ast.NodeKind, number | undefined];
@@ -33,9 +34,11 @@ async function collectAbridgeNodeFromAst(text: string): Promise<ReadonlyArray<Ab
     const state: CollectAbridgeNodeState = {
         locale: DefaultLocale,
         result: [],
+        maybeCancellationToken: undefined,
+        traceManager: new NoOpTraceManager(),
     };
 
-    const triedTraverse: Traverse.TriedTraverse<AbridgedNode[]> = Traverse.tryTraverseAst<
+    const triedTraverse: Traverse.TriedTraverse<AbridgedNode[]> = await Traverse.tryTraverseAst<
         CollectAbridgeNodeState,
         AbridgedNode[]
     >(
@@ -66,9 +69,11 @@ async function assertGetNthNodeOfKind<N extends Language.Ast.TNode>(
         nodeKind,
         nthCounter: 0,
         nthRequired,
+        maybeCancellationToken: undefined,
+        traceManager: new NoOpTraceManager(),
     };
 
-    const triedTraverse: Traverse.TriedTraverse<Language.Ast.TNode | undefined> = Traverse.tryTraverseAst<
+    const triedTraverse: Traverse.TriedTraverse<Language.Ast.TNode | undefined> = await Traverse.tryTraverseAst<
         NthNodeOfKindState,
         Language.Ast.TNode | undefined
     >(
@@ -86,11 +91,13 @@ async function assertGetNthNodeOfKind<N extends Language.Ast.TNode>(
     return Assert.asDefined(triedTraverse.value) as N;
 }
 
-function collectAbridgeNodeVisit(state: CollectAbridgeNodeState, node: Language.Ast.TNode): void {
+// eslint-disable-next-line require-await
+async function collectAbridgeNodeVisit(state: CollectAbridgeNodeState, node: Language.Ast.TNode): Promise<void> {
     state.result.push([node.kind, node.maybeAttributeIndex]);
 }
 
-function nthNodeVisit(state: NthNodeOfKindState, node: Language.Ast.TNode): void {
+// eslint-disable-next-line require-await
+async function nthNodeVisit(state: NthNodeOfKindState, node: Language.Ast.TNode): Promise<void> {
     if (node.kind === state.nodeKind) {
         state.nthCounter += 1;
 
@@ -100,7 +107,8 @@ function nthNodeVisit(state: NthNodeOfKindState, node: Language.Ast.TNode): void
     }
 }
 
-function nthNodeEarlyExit(state: NthNodeOfKindState, _: Language.Ast.TNode): boolean {
+// eslint-disable-next-line require-await
+async function nthNodeEarlyExit(state: NthNodeOfKindState, _: Language.Ast.TNode): Promise<boolean> {
     return state.nthCounter === state.nthRequired;
 }