From 03b9cdce13bb99a55a99db46bb7dc3196717769f Mon Sep 17 00:00:00 2001
From: david072 <david.g.ganz@gmail.com>
Date: Thu, 19 Dec 2024 16:01:35 +0100
Subject: [PATCH 1/2] Add explanatory comments to LSP implementation
---
README.md | 5 +
server/src/server.ts | 140 +++++++++++------
server/src/tas-script/diagnosticCollector.ts | 1 +
server/src/tas-script/otherCompletion.ts | 2 +
server/src/tas-script/tasScript.ts | 156 +++++++++++++++----
server/src/tas-script/tasTool.ts | 41 +++--
server/src/tas-script/tokenizer.ts | 34 +++-
7 files changed, 289 insertions(+), 90 deletions(-)
diff --git a/README.md b/README.md
index a847286..db2b7f7 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,11 @@ Syntax highlighting, snippets and autocompletion for the Portal 2 TAS files, usi
1. Go to the [Marketplace](https://marketplace.visualstudio.com/items?itemName=Portal2SpeedrunningHub.p2tas) or search for "Portal 2 TAS Tools" in the extensions tab in Visual Studio Code
2. Press "Install"
+## Building
+
+1. Install packages using `npm install` in the root directory
+2. Run the extension and language server using the "Run and Debug" feature of VSCode. Selecting "Client + Server" will create a new VSCode session with the extension installed, as well as start the language server and connect a debugger to both.
+
## Release Notes
### 1.4.2
diff --git a/server/src/server.ts b/server/src/server.ts
index 37114c6..2f51b63 100644
--- a/server/src/server.ts
+++ b/server/src/server.ts
@@ -55,6 +55,10 @@ var settings: Settings = defaultSettings;
connection.onDidChangeConfiguration(_ => pullSettings());
+/**
+ * Gets settings from client configuration.
+ * In VSCode, these can be found in the Settings tab (or the configuration JSON file).
+ */
async function pullSettings() {
const configuration = await connection.workspace.getConfiguration({ section: "p2tasLanguageServer" });
@@ -82,20 +86,24 @@ rawDocuments.onDidOpen((params) => {
});
rawDocuments.onDidChangeContent((params) => {
+ // parse the script again to collect diagnostics and general script information
+ // could to incremental parsing here, probably overkill for our usecase though
const diagnostics = documents.get(params.document.uri)?.parse(params.document.getText());
if (diagnostics && settings.doErrorChecking) connection.sendDiagnostics({ uri: params.document.uri, diagnostics });
});
rawDocuments.onDidClose((params) => { documents.delete(params.document.uri); });
-// FIXME: This needs to check if we are in a comment / skip comments on the way of finding the tool.
-// One idea might be to break when we find a comment open token in "getToolAndArguments",
-// and advance to after the comment if we find a "*/".
+// FIXME: We currently also suggest tool arguments when in a multiline comment. We should check whether
+// the cursor is in a multiline comment and don't suggest anything.
connection.onCompletion((params: TextDocumentPositionParams): CompletionItem[] => {
const script = documents.get(params.textDocument.uri);
if (script === undefined) return [];
const line = script.lines.get(params.position.line);
+ // the line of the cursor is not present in our script (happens e.g. when the user inserts empty lines below the last framebulk)
+ // => suggest start, version, repeat, end
+ // FIXME: Don't suggest version/start if they are already present in the script.
if (line === undefined) {
return [versionCompletion, startCompletion, repeatCompletion, endCompletion].map((val) => {
return {
@@ -108,8 +116,8 @@ connection.onCompletion((params: TextDocumentPositionParams): CompletionItem[] =
};
});
}
+ // suggest tools and arguments for framebulks with 4 pipes, when the cursor is positioned after the last pipe
else if (line.type === LineType.Framebulk) {
- // If we don't have 4 pipes, dont suggest
if (line.tokens.filter(tok => tok.type === TokenType.Pipe).length !== 4)
return [];
@@ -118,12 +126,14 @@ connection.onCompletion((params: TextDocumentPositionParams): CompletionItem[] =
return completeToolAndArguments(line, params.position.character);
}
+ // suggest tools and arguments to tool bulks when after the last '>'
else if (line.type === LineType.ToolBulk) {
const angleIndex = line.lineText.lastIndexOf('>');
if (params.position.character < angleIndex) return [];
return completeToolAndArguments(line, params.position.character);
}
+ // suggest "start" line parameters
else if (line.type === LineType.Start) {
const [toolName, encounteredWords] = getToolAndArguments(params.position.character, line.tokens);
if (toolName !== "start") return [];
@@ -152,11 +162,12 @@ connection.onCompletion((params: TextDocumentPositionParams): CompletionItem[] =
return [];
});
+/** Resolve `CompletionItems` for the tool focused by the cursor in line `line` and column `character`. */
function completeToolAndArguments(line: ScriptLine, character: number): CompletionItem[] {
const [toolName, encounteredWords] = getToolAndArguments(character, line.tokens);
+ // if no tool was found (e.g. the cursor is right behind a '|'), suggest tools
if (toolName.length === 0) {
- // Complete tool
return Object.entries(TASTool.tools).map(([key, value]) => {
return {
label: key,
@@ -168,14 +179,14 @@ function completeToolAndArguments(line: ScriptLine, character: number): Completi
};
});
}
+ // since a tool was found, suggest its arguments
else {
- // Complete tool arguments
- // Check if tool exists
if (!TASTool.tools.hasOwnProperty(toolName)) return [];
if (encounteredWords.includes("off")) return [];
const tool = TASTool.tools[toolName];
const result: CompletionItem[] = [];
+ // add "off" argument suggestion if the tool supports it and no other arguments have been given
if (encounteredWords.length === 0 && tool.hasOff) {
result.push({
label: "off",
@@ -187,6 +198,7 @@ function completeToolAndArguments(line: ScriptLine, character: number): Completi
});
}
+ // suggest arguments that haven't been given yet
const toolArguments = tool.arguments;
result.push(...toolArguments
.filter((arg) => {
@@ -206,7 +218,14 @@ function completeToolAndArguments(line: ScriptLine, character: number): Completi
}
}
+/**
+ * From the given character index, go back through the line's tokens to find the corresponding tool and its
+ * arguments up to this point.
+ */
+// TODO: It might be better to have the parser extract this information from the script instead of searching
+// through the tokens here.
function getToolAndArguments(character: number, tokens: Token[]): [string, string[]] {
+ // the words we found while going through the line backwards to search for the tool name
var encounteredWords: string[] = [];
var tool = "";
@@ -214,8 +233,13 @@ function getToolAndArguments(character: number, tokens: Token[]): [string, strin
outer: {
while (--index >= 0) {
const token = tokens[index];
+ // skip tokens after the given character position
if (character < token.end) continue;
+ // A section (tool section of a framebulk/tool bulk, tool invocation) has ended. Since we're only
+ // interested in the tool and arguments the cursor is "in", stop here.
+ // This is needed when the cursor is right behind a '|' for example.
if (token.type === TokenType.Pipe || token.type === TokenType.Semicolon || token.type === TokenType.DoubleRightAngle) break outer;
+ // reached the start of the line
if (index - 1 < 0) {
tool = token.text;
break;
@@ -225,6 +249,7 @@ function getToolAndArguments(character: number, tokens: Token[]): [string, strin
case TokenType.String:
encounteredWords.push(token.text);
break;
+ // we have reached the end of a section
case TokenType.Semicolon:
case TokenType.Pipe:
case TokenType.DoubleRightAngle:
@@ -247,43 +272,51 @@ connection.onHover((params, cancellationToken, workDoneProgressReporter, resultP
if (line === undefined) return undefined;
if (line.type === LineType.Framebulk || line.type === LineType.ToolBulk) {
+ // find hovered token
for (var i = 0; i < line.tokens.length; i++) {
const token = line.tokens[i];
- if (params.position.character >= token.start && params.position.character <= token.end) {
- if (token.type === TokenType.Number) {
- if (i + 1 >= line.tokens.length || (line.tokens[i + 1].type !== TokenType.RightAngle && line.tokens[i + 1].type !== TokenType.DoubleRightAngle)) continue;
- return { contents: [`Tick: ${line.tick}`] };
+ if (params.position.character < token.start || params.position.character > token.end) continue;
+
+ // if we are hovering the tick number at the start of the framebulk, show the absolute tick
+ if (token.type === TokenType.Number) {
+ if (i + 1 >= line.tokens.length || (line.tokens[i + 1].type !== TokenType.RightAngle && line.tokens[i + 1].type !== TokenType.DoubleRightAngle)) continue;
+ return { contents: [`Tick: ${line.tick}`] };
+ }
+
+ if (token.type !== TokenType.String) continue;
+
+ // show information tools or tool arguments
+ // TODO: This currently doesn't check which tool the hovered argument belongs to. By doing something
+ // similar to completeToolAndArguments to find out the tool in question, this could be improved.
+ // In addition, that might allow us to show information on "off" arguments for example.
+ const hoveredWord = token.text;
+ for (const tool of Object.keys(TASTool.tools)) {
+ if (tool === hoveredWord) {
+ return {
+ contents: {
+ kind: MarkupKind.Markdown,
+ value: TASTool.tools[tool].description
+ }
+ };
}
- else if (token.type !== TokenType.String) continue;
- const hoveredWord = token.text;
- for (const tool of Object.keys(TASTool.tools)) {
- if (tool === hoveredWord) {
+ for (const argument of TASTool.tools[tool].arguments) {
+ if (argument.type !== TokenType.String) continue;
+ if (argument.text === hoveredWord) {
+ if (argument.description === undefined) break;
return {
contents: {
kind: MarkupKind.Markdown,
- value: TASTool.tools[tool].description
+ value: argument.description,
}
};
}
-
- for (const argument of TASTool.tools[tool].arguments) {
- if (argument.type !== TokenType.String) continue;
- if (argument.text === hoveredWord) {
- if (argument.description === undefined) break;
- return {
- contents: {
- kind: MarkupKind.Markdown,
- value: argument.description,
- }
- };
- }
- }
}
}
}
}
else if (line.type === LineType.RepeatStart) {
+ // show information on the "repeat" keyword when hovering it
if (line.tokens.length > 0 && line.tokens[0].type === TokenType.String && line.tokens[0].text === "repeat" &&
params.position.character >= line.tokens[0].start && params.position.character <= line.tokens[0].end) {
return {
@@ -295,29 +328,31 @@ connection.onHover((params, cancellationToken, workDoneProgressReporter, resultP
}
}
else if (line.type === LineType.Start) {
+ // find hovered token and show information on the "start" keyword or its arguments
for (const token of line.tokens) {
- if (params.position.character >= token.start && params.position.character <= token.end) {
- if (token.text === "start")
- return {
- contents: {
- kind: MarkupKind.Markdown,
- value: startCompletion.description
- }
- };
+ if (params.position.character < token.start || params.position.character > token.end) continue;
- if (startTypes.hasOwnProperty(token.text))
- return {
- contents: {
- kind: MarkupKind.Markdown,
- value: startTypes[token.text].description
- }
- };
- }
+ if (token.text === "start")
+ return {
+ contents: {
+ kind: MarkupKind.Markdown,
+ value: startCompletion.description
+ }
+ };
+
+ if (startTypes.hasOwnProperty(token.text))
+ return {
+ contents: {
+ kind: MarkupKind.Markdown,
+ value: startTypes[token.text].description
+ }
+ };
}
return undefined;
}
else if (line.type === LineType.End) {
+ // show information on the "end" keyword when hovering it
if (line.tokens.length > 0 && line.tokens[0].type === TokenType.String && line.tokens[0].text === "end" &&
params.position.character >= line.tokens[0].start && params.position.character <= line.tokens[0].end) {
return {
@@ -329,6 +364,7 @@ connection.onHover((params, cancellationToken, workDoneProgressReporter, resultP
}
}
else if (line.type === LineType.Version) {
+ // show information on the "version" keyword when hovering it
if (line.tokens.length > 0 && line.tokens[0].type === TokenType.String && line.tokens[0].text === "version" &&
params.position.character >= line.tokens[0].start && params.position.character <= line.tokens[0].end) {
return {
@@ -341,6 +377,10 @@ connection.onHover((params, cancellationToken, workDoneProgressReporter, resultP
}
});
+/**
+ * Returns the active tools on request of the client.
+ * Format: <tool name>/<from line>/<start column>/<end column>[/<ticks remaining if possible>]
+ */
connection.onRequest("p2tas/activeTools", (params: [any, number]) => {
const [uri, lineNumber] = params;
@@ -351,9 +391,10 @@ connection.onRequest("p2tas/activeTools", (params: [any, number]) => {
return line.activeTools.map(tool =>
`${tool.tool}/${tool.fromLine}/${tool.startCol}/${tool.endCol}` + (tool.ticksRemaining ? `/${tool.ticksRemaining}` : "")
- ).join(","); // e.g. 'autoaim/0,setang/1/5'
+ ).join(",");
});
+/** Returns the absolute tick of the given line on request of the client. */
connection.onRequest("p2tas/lineTick", (params: [any, number]) => {
const [uri, lineNumber] = params;
@@ -363,6 +404,7 @@ connection.onRequest("p2tas/lineTick", (params: [any, number]) => {
return script.lines.get(lineNumber)?.tick || "";
});
+/** Returns the line at the line with the given tick, or the one before on request of the client. */
connection.onRequest("p2tas/tickLine", (params: [any, number]) => {
const [uri, tick] = params;
@@ -380,6 +422,7 @@ connection.onRequest("p2tas/tickLine", (params: [any, number]) => {
return lastLine == -1 ? "" : lastLine;
});
+/** Toggles the given line's tick type (absolute <=> relative) on request of the client. */
connection.onRequest("p2tas/toggleLineTickType", (params: [any, number]) => {
const [uri, lineNumber] = params;
@@ -395,9 +438,10 @@ connection.onRequest("p2tas/toggleLineTickType", (params: [any, number]) => {
let previousLine: ScriptLine | undefined = undefined;
let prevLineNumber = lineNumber;
+ // Find the previous line
while (previousLine === undefined) {
prevLineNumber--;
-
+
if (prevLineNumber < 0) return line.lineText;
previousLine = script.lines.get(prevLineNumber)
@@ -409,6 +453,7 @@ connection.onRequest("p2tas/toggleLineTickType", (params: [any, number]) => {
// Invalid line format
if (line.tokens[0].type !== TokenType.Number) return line.lineText;
+ // Reformat the line to use the new tick format
const newTickSection = `+${line.tick - previousLine.tick}`;
// everything before the number -|relative tick -|everything after the tick
return `${line.lineText.substring(0, line.tokens[0].start)}${newTickSection}${line.lineText.substring(line.tokens[0].end).replace(/\r|\n/, "")}`;
@@ -418,6 +463,7 @@ connection.onRequest("p2tas/toggleLineTickType", (params: [any, number]) => {
// Invalid line format
if (line.tokens[0].type !== TokenType.Plus || line.tokens[1].type !== TokenType.Number) return line.lineText;
+ // We already have the absolute tick of every line parsed out, so we just need to reformat the line to use it
const newTickSection = `${line.tick}`;
// everything before the plus -|everything after the plus -|absolute tick -|everything after the tick (remove new line)
return `${line.lineText.substring(0, line.tokens[0].start)}${line.lineText.substring(line.tokens[0].end, line.tokens[1].start)}${newTickSection}${line.lineText.substring(line.tokens[1].end).replace(/\r|\n/, "")}`;
diff --git a/server/src/tas-script/diagnosticCollector.ts b/server/src/tas-script/diagnosticCollector.ts
index afff423..c5cb8a3 100644
--- a/server/src/tas-script/diagnosticCollector.ts
+++ b/server/src/tas-script/diagnosticCollector.ts
@@ -1,5 +1,6 @@
import { Diagnostic, DiagnosticSeverity } from "vscode-languageserver/node";
+/// Helper to collect diagnostics while parsing to return to the client.
export class DiagnosticCollector {
private static instance: DiagnosticCollector;
private diagnostics: Diagnostic[] = [];
diff --git a/server/src/tas-script/otherCompletion.ts b/server/src/tas-script/otherCompletion.ts
index 1473367..1460c92 100644
--- a/server/src/tas-script/otherCompletion.ts
+++ b/server/src/tas-script/otherCompletion.ts
@@ -1,3 +1,5 @@
+//! Completion/hover descriptions for non-tool items (start, version, repeat/end statements).
+
interface CompletionDefinition {
[name: string]: {
description: string
diff --git a/server/src/tas-script/tasScript.ts b/server/src/tas-script/tasScript.ts
index c848997..ec66d91 100644
--- a/server/src/tas-script/tasScript.ts
+++ b/server/src/tas-script/tasScript.ts
@@ -3,17 +3,34 @@ import { DiagnosticCollector } from "./diagnosticCollector";
import { TASTool } from "./tasTool";
import { Token, tokenize, TokenType } from "./tokenizer";
+/**
+ * The state of the parser. The parser advances the state in the following way (trying to accept
+ * the respective tokens or falling back to defaults): \
+ * Version -> Start -> RngManip -> Framebulks
+ */
enum ParserState {
Version, Start, RngManip, Framebulks
}
+/**
+ * Parser for a TAS script, collecting useful information for the LSP.
+ * Due to the nature of TAS scripts, this parser works as a state machine, first expecting a version
+ * statement, then a start statement and finally framebulks.
+ *
+ * NOTE: The files this operates on are, in the majority of cases, incorrect/incomplete as the user is
+ * still working on them. This parser thus needs to be able to recover from errors and continue
+ * parsing as much as possible, even if errors have been encountered.
+ */
export class TASScript {
+ /** Default assumed script version */
+ readonly DEFAULT_VERSION = 7;
+
fileText = "";
- // Map of line# -> ScriptLine
+ /** Map of line number to ScriptLine */
lines = new Map<number, ScriptLine>();
- scriptVersion = 4;
+ scriptVersion = this.DEFAULT_VERSION;
tokens: Token[][] = [];
lineIndex = 0;
@@ -27,6 +44,7 @@ export class TASScript {
return entries[i][1];
}
+ /** Parses the given file, extracting useful information into the `lines` field and returns diagnostic information. */
parse(fileText?: string): Diagnostic[] {
new DiagnosticCollector();
@@ -42,9 +60,10 @@ export class TASScript {
[this.tokens, lines] = tokenize(this.fileText);
var state = ParserState.Version;
+ /** Used to return diagnostic for when the first framebulk is relative. */
var isFirstFramebulk = true;
- // Stack for nested repeats; stores iterations, starting tick, lineIndex of "repeat"
+ /** Stack for nested repeats; stores iterations, starting tick, lineIndex of "repeat". */
var repeats: [number, number, number][] = [];
while (this.lineIndex < this.tokens.length) {
@@ -57,14 +76,16 @@ export class TASScript {
const currentLineText = lines[currentLine];
switch (state) {
+ // Try to accept `version <number>`, falling back to DEFAULT_VERSION
case ParserState.Version:
this.expectText("Expected version", "version");
- this.scriptVersion = this.expectNumber("Invalid version", 1, 2, 3, 4, 5, 6, 7) ?? 7;
+ this.scriptVersion = this.expectNumber("Invalid version", 1, 2, 3, 4, 5, 6, 7) ?? this.DEFAULT_VERSION;
this.expectCount("Ignored parameters", 2);
this.lines.set(currentLine, new ScriptLine(currentLineText, 0, false, LineType.Version, [], this.tokens[this.lineIndex]));
state = ParserState.Start;
break;
+ // Try to accept `start [next] now` or `start [next] map/save/cm <map/save>`
case ParserState.Start:
this.expectText("Expected start", "start");
const startType = this.expectText("Expected start type", "map", "save", "cm", "now", "next");
@@ -97,6 +118,7 @@ export class TASScript {
}
};
+ // Accept start statement + its arguments, as well as a map/save string if necessary
if (startType !== undefined) {
if (startType === "next") {
const startType = this.expectText("Expected start type", "map", "save", "cm", "now");
@@ -119,6 +141,7 @@ export class TASScript {
this.lines.set(currentLine, new ScriptLine(currentLineText, 0, false, LineType.Start, [], this.tokens[this.lineIndex]));
state = ParserState.RngManip;
break;
+ // Try to accept `rngmanip <path>`
case ParserState.RngManip:
const token = this.next("Expected framebulks or rngmanip");
if (token === undefined) break;
@@ -131,9 +154,12 @@ export class TASScript {
}
state = ParserState.Framebulks;
break;
+ // Try to accept framebulks for the rest of the file
case ParserState.Framebulks:
const prevLine = this.previousLine()!;
+ // Handle repeat/end
+ // i.e. `repeat <iterations> \n <framebulks> \n end`
if (this.isNextType(TokenType.String)) {
const token = this.tokens[this.lineIndex][this.tokenIndex - 1];
if (token.text === "repeat") {
@@ -143,29 +169,38 @@ export class TASScript {
if (repeatCount !== undefined)
repeats.push([repeatCount, tick, this.lineIndex]);
this.lines.set(currentLine, new ScriptLine(currentLineText, tick, false, LineType.RepeatStart, prevLine.activeTools, this.tokens[this.lineIndex]));
- break;
}
+ // Handle repeat "end". The tick of the line that contains the "end" keyword is
+ // the tick that is reached after the loop is over.
else if (token.text === "end") {
var endTick = 0;
+ // Invalid end (no preceeding "repeat").
if (repeats.length === 0) {
DiagnosticCollector.addDiagnostic(token.line, token.start, token.end, "End outside of loop")
endTick = prevLine.tick;
}
+ // Calculate loop end tick. Since we processed the loop content, the last line of the repeat block will have the tick
+ // that is reached after one iteration. Therefore, we have to multiply the duration of the loop with the iterations
+ // minus 1 to get the remaining duration of the loop after one iteration. The duration of the loop can be calculated
+ // using the tick of the "repeat" line.
else {
const [iterations, startingTick] = repeats.pop()!;
const repeatEnd = prevLine.tick;
endTick = prevLine.tick + (repeatEnd - startingTick) * (iterations - 1);
}
this.lines.set(currentLine, new ScriptLine(currentLineText, endTick, false, LineType.End, prevLine.activeTools, this.tokens[this.lineIndex]));
- break;
}
else {
DiagnosticCollector.addDiagnostic(token.line, token.start, token.end, "Unexpected token");
this.lines.set(currentLine, new ScriptLine(currentLineText, -1, false, LineType.Framebulk, prevLine.activeTools, this.tokens[this.lineIndex]));
- break;
}
+
+ break;
}
+ // Parse framebulks
+
+ // Check whether the framebulk is relative ('+' before the tick); the first framebulk cannot be relative!
const maybePlus = this.currentToken();
const isRelative = this.isNextType(TokenType.Plus);
if (isRelative && isFirstFramebulk)
@@ -174,17 +209,19 @@ export class TASScript {
const tickToken = this.currentToken();
const tick = this.expectNumber("Expected tick") || 0;
- const angleToken = this.expectType("Expected '>' or '>>'", TokenType.RightAngle, TokenType.DoubleRightAngle);
- const isToolBulk = angleToken !== undefined && angleToken.type === TokenType.DoubleRightAngle;
-
- var activeTools = prevLine.activeTools.map((val) => val.copy());
-
var absoluteTick = isRelative ? prevLine.tick + tick : tick;
const previousLineTick = prevLine.tick;
if ((prevLine.type === LineType.Framebulk || prevLine.type === LineType.ToolBulk) && absoluteTick <= previousLineTick)
DiagnosticCollector.addDiagnostic(tickToken.line, tickToken.start, tickToken.end, `Expected tick greater than ${previousLineTick}`)
+ const angleToken = this.expectType("Expected '>' or '>>'", TokenType.RightAngle, TokenType.DoubleRightAngle);
+ const isToolBulk = angleToken !== undefined && angleToken.type === TokenType.DoubleRightAngle;
+
+ // Deep copy the previous line's active tools, so that we can modify it independently
+ var activeTools = prevLine.activeTools.map((val) => val.copy());
+
+ // Update `activeTools` by updating `ticksRemaining` and removing tools that are no longer active
for (var i = 0; i < activeTools.length; i++) {
if (activeTools[i].ticksRemaining === undefined) continue;
activeTools[i].ticksRemaining! -= absoluteTick - previousLineTick;
@@ -226,10 +263,6 @@ export class TASScript {
// Tools field
this.parseToolsField(activeTools);
- if (this.tokens[this.lineIndex].length > this.tokenIndex) {
- const token = this.currentToken();
- DiagnosticCollector.addDiagnosticToLine(token.line, token.end, "Unexpected tokens");
- }
}
// Sort tools according to their priority index from SAR if version >= 3
@@ -252,10 +285,16 @@ export class TASScript {
break;
}
+ if (this.tokenIndex < this.tokens[this.lineIndex].length) {
+ const lastToken = this.currentToken();
+ DiagnosticCollector.addDiagnosticToLine(lastToken.line, lastToken.start, "Unexpected tokens");
+ }
+
this.tokenIndex = 0;
this.lineIndex++;
}
+ // No tokens left => check what the user is missing
if (state === ParserState.Version) {
DiagnosticCollector.addDiagnosticToLine(integer.MAX_VALUE, 0, "Expected version");
}
@@ -279,12 +318,15 @@ export class TASScript {
private parseButtonsField() {
if (this.tokens[this.lineIndex].length <= this.tokenIndex) return;
if (this.isNextType(TokenType.Pipe)) {
- this.tokenIndex--; // Decrement token index for the check to pass after this function
+ // Decrement `tokenIndex` to allow the caller to accept the pipe themselves, since this is
+ // simply the indication for us that there is nothing to do here.
+ this.tokenIndex--;
return;
}
const buttons = this.expectText("Expected buttons") || "";
const token = this.currentToken();
+ // Collect button information, each button being followed by an optional hold duration (e.g. "J1")
for (var i = 0; i < buttons.length; i++) {
var button = buttons[i];
var wasUpper = false;
@@ -309,10 +351,12 @@ export class TASScript {
}
}
+ /** Parse tools and their arguments starting at `this.tokenIndex` and insert them into `activeTools`. */
private parseToolsField(activeTools: TASTool.Tool[]) {
while (this.tokens[this.lineIndex].length > this.tokenIndex) {
if (this.isNextType(TokenType.Semicolon)) continue;
+ // Expect and verify the tool's name
const toolName = this.expectText("Expected tool");
const toolNameToken = this.tokens[this.lineIndex][this.tokenIndex - 1];
if (toolName === undefined || !TASTool.tools.hasOwnProperty(toolName)) {
@@ -321,31 +365,41 @@ export class TASScript {
continue;
}
+ // A tool currently always has to have at least 1 argument
if (this.isNextType(TokenType.Semicolon) || this.tokenIndex >= this.tokens[this.lineIndex].length) {
DiagnosticCollector.addDiagnosticToLine(toolNameToken.line, toolNameToken.end, "Expected arguments");
this.tokenIndex++;
continue;
}
+ // Remove the tool from `activeTools` if it is already present, as we'll re-add it with updated parameters
const toolIndex = activeTools.findIndex((val) => val.tool === toolName || val.tool === `(${toolName})`);
if (toolIndex !== -1) activeTools.splice(toolIndex, 1);
const tool = TASTool.tools[toolName];
const firstArgument = this.tokens[this.lineIndex][this.tokenIndex];
+ // If the tool has an "off" argument, it should be the first and only argument given to the tool
if (tool.hasOff && firstArgument.type === TokenType.String && firstArgument.text === "off") {
this.tokenIndex++;
if (this.tokenIndex >= this.tokens[this.lineIndex].length) return;
+ // Consume everything up to the next semicolon if there are more arguments after the "off" argument
+ // and emit diagnostics accordingly
if (!this.isNextType(TokenType.Semicolon)) {
const token = this.currentToken();
this.moveToNextSemicolon();
- const lastToken = this.tokens[this.lineIndex][this.tokenIndex - (this.tokenIndex === this.tokens[this.lineIndex].length ? 1 : 2)];
- DiagnosticCollector.addDiagnostic(token.line, token.start, lastToken.end, "Expected ';'");
+ DiagnosticCollector.addDiagnosticToLine(token.line, token.start, "Expected ';'");
}
continue;
}
var toolDuration: number | undefined = undefined;
- if (tool.isOrderDetermined) blk: {
+ if (tool.hasFixedOrder) blk: {
+ // Expect the tool arguments in the order they were defined in `tool.arguments`, as they have to appear in this order.
+
+ /**
+ * Arguments that need to come next (used to parse argument children before other arguments). This is emptied first,
+ * before continuing with `tool.arguments`
+ */
var queue: TASTool.ToolArgument[] = [];
var i = 0;
@@ -383,7 +437,9 @@ export class TASScript {
const token = this.tokens[this.lineIndex][this.tokenIndex - 1];
if (arg.text! === token.text) break inner;
+ // The argument wasn't matched => expect the argument's otherwiseChildren next
if (arg.otherwiseChildren !== undefined) queue.splice(0, 0, ...arg.otherwiseChildren!);
+ // Backtrack, since the argument wasn't matched
this.tokenIndex--;
continue;
}
@@ -391,6 +447,7 @@ export class TASScript {
this.validateArgument(arg);
}
+ // Extract the tools duration
if (i === tool.durationIndex)
toolDuration = +this.tokens[this.lineIndex][this.tokenIndex - 1].text;
@@ -398,10 +455,12 @@ export class TASScript {
continue;
}
+ // The argument wasn't matched => expect the argument's otherwiseChildren next
if (arg.otherwiseChildren !== undefined) queue.splice(0, 0, ...arg.otherwiseChildren!);
}
}
+ // Update `activeTools` with the new tool
if (tool.durationIndex === -1) {
activeTools.push(new TASTool.Tool(
toolName,
@@ -410,6 +469,9 @@ export class TASScript {
this.tokens[this.lineIndex][this.tokenIndex - 1].end,
));
} else {
+ // FIXME: autoaim's duration argument is optional, and if not supplied, autoaim will stay active until manually
+ // turned off. However, since it has a durationIndex, we are not handling it above. This should be handled
+ // in a more general way.
if (toolName === "autoaim" || toolDuration !== undefined) {
activeTools.push(new TASTool.Tool(
toolName,
@@ -421,7 +483,9 @@ export class TASScript {
}
}
+ // Since we have matched all arguments of the tool, the tool usage should end here (either by semicolon or by the end of the line)
if (this.tokenIndex >= this.tokens[this.lineIndex].length) return;
+ // If the next token is not a semicolon, there is more invalid text after the tool has been given all its arguments
if (!this.isNextType(TokenType.Semicolon)) {
const token = this.currentToken();
this.moveToNextSemicolon();
@@ -431,23 +495,30 @@ export class TASScript {
}
}
else {
+ // Accept the tool arguments in any order
while (this.tokenIndex < this.tokens[this.lineIndex].length && this.tokens[this.lineIndex][this.tokenIndex].type !== TokenType.Semicolon) {
- const argument = this.currentToken();
+ const argumentToken = this.currentToken();
blk: {
- for (const arg of tool.arguments) {
- if (arg.type === argument.type) {
- if (argument.type === TokenType.String) {
- if (arg.text !== undefined && arg.text === argument.text)
+ // Try to find an argument that matches
+ for (const toolArg of tool.arguments) {
+ if (toolArg.type === argumentToken.type) {
+ if (argumentToken.type === TokenType.String) {
+ if (toolArg.text !== undefined && toolArg.text === argumentToken.text)
break blk;
}
- else if (arg.type === TokenType.Number && arg.unit !== undefined) {
+ else if (toolArg.type === TokenType.Number && toolArg.unit !== undefined) {
if (this.tokenIndex + 1 < this.tokens[this.lineIndex].length) {
+ // Increment the `tokenIndex` briefly to check the token's type after the argument
+ // (`tokenIndex` is pointing at the number)
this.tokenIndex++;
if (this.isNextType(TokenType.String)) {
this.tokenIndex--;
const unitToken = this.tokens[this.lineIndex][this.tokenIndex];
- if (unitToken.text === arg.unit) break blk; // TODO: Maybe more information here?
+ // TODO: Emit diagnostics for incorrect units.
+ if (unitToken.text === toolArg.unit) break blk;
}
+ // Decrement again, since the argument didn't match and we need to check the next
+ // argument using the loop
this.tokenIndex--;
}
}
@@ -455,13 +526,16 @@ export class TASScript {
}
}
- DiagnosticCollector.addDiagnostic(argument.line, argument.start, argument.end, "Invalid argument");
+ // Couldn't find a matching argument
+ DiagnosticCollector.addDiagnostic(argumentToken.line, argumentToken.start, argumentToken.end, "Invalid argument");
}
this.tokenIndex++;
}
this.tokenIndex++;
+ // Update `activeTools` with the new tool
activeTools.push(new TASTool.Tool(
+ // TODO: No special case for "decel" tool?!
toolName !== "decel" ? toolName : "(decel)",
this.lineIndex,
toolNameToken.start,
@@ -472,6 +546,10 @@ export class TASScript {
}
}
+ /**
+ * Emits diagnostics for the argument before `tokenIndex`.
+ * Checks that the text matches the given argument or that the corresponding number has the right unit attached to it.
+ */
private validateArgument(arg: TASTool.ToolArgument) {
const argumentToken = this.tokens[this.lineIndex][this.tokenIndex - 1];
if (arg.type === TokenType.String && arg.text !== undefined) {
@@ -523,6 +601,10 @@ export class TASScript {
}
}
+ /**
+ * Return a token if it matches any of `types`, or undefined, in which case a diagnostic with the
+ * given `errorText` will be created at the erroneous token.
+ */
private expectType(errorText: string, ...types: TokenType[]): Token | undefined {
const token = this.next(errorText);
if (token === undefined) return;
@@ -554,6 +636,10 @@ export class TASScript {
return line[this.tokenIndex++];
}
+ /**
+ * Return a string if the next token is of `TokenType.String` and matches a text from `text`, or undefined,
+ * in which case a diagnostic with the given `errorText` will be created at the erroneous token.
+ */
private expectText(errorText: string, ...text: string[]): string | undefined {
const token = this.next(errorText);
if (token === undefined) return;
@@ -571,6 +657,10 @@ export class TASScript {
return token.text;
}
+ /**
+ * Return a number if the next token is of `TokenType.Number` and matches a number from `number`, or undefined,
+ * in which case a diagnostic with the given `errorText` will be created at the erroneous token.
+ */
private expectNumber(errorText: string, ...number: number[]): number | undefined {
const token = this.next();
if (token === undefined) return;
@@ -590,13 +680,16 @@ export class TASScript {
return num;
}
+ /**
+ * Expect there to be `count` tokens in the current line, or emit a diagnostic from the last valid (fitting in count) token
+ * to the end of the line.
+ */
private expectCount(errorText: string, count: number) {
if (this.tokens[this.lineIndex].length === count) return;
const firstInvalidToken = this.tokens[this.lineIndex][count];
if (firstInvalidToken === undefined) return;
DiagnosticCollector.addDiagnosticToLine(firstInvalidToken.line, firstInvalidToken.start, errorText);
}
-
}
export enum LineType {
@@ -609,12 +702,17 @@ export enum LineType {
ToolBulk,
}
+/** Information about a line in the TAS script. */
export class ScriptLine {
constructor(
+ /** The raw text of the line. */
public lineText: string,
+ /** The absolute tick of the line in the script. */
public tick: number,
+ /** Whether the tick of the line is relative. */
public isRelative: boolean,
public type: LineType,
+ /** Which tools are active at this line. */
public activeTools: TASTool.Tool[],
public tokens: Token[],
) { }
diff --git a/server/src/tas-script/tasTool.ts b/server/src/tas-script/tasTool.ts
index 5a1e86d..28b65c7 100644
--- a/server/src/tas-script/tasTool.ts
+++ b/server/src/tas-script/tasTool.ts
@@ -4,8 +4,11 @@ export namespace TASTool {
export class Tool {
constructor(
public tool: string,
+ /** The line at which the tool was invoked. */
public fromLine: number,
+ /** The column on the `fromLine` at which the tool invocation starts (first character of the tool name). */
public startCol: number,
+ /** The column on the `fromLine` at which the tool invocation ends (character after the last argument). */
public endCol: number,
public ticksRemaining?: number,
) { }
@@ -17,12 +20,19 @@ export namespace TASTool {
interface ToolDefinition {
[name: string]: {
- readonly isOrderDetermined: boolean,
+ /** Whether the arguments of this tool have a fixed order. */
+ readonly hasFixedOrder: boolean,
+ /**
+ * Whether the tool has an "off" argument. This is treated separately, as it should always
+ * appear on its own and should not be suggested if other arguments are present.
+ */
readonly hasOff: boolean,
+ /** The index of the argument in `arguments` that defines for how long the tool runs. */
readonly durationIndex: number,
readonly arguments: ToolArgument[],
readonly description: string,
- readonly index: number, // priority index from SAR (- 1) (used only in version >= 3)
+ /** Index of the tool in SAR's execution order (minus 1) (used only in version >= 3). */
+ readonly index: number,
}
}
@@ -31,16 +41,25 @@ export namespace TASTool {
readonly type: TokenType,
readonly required: boolean,
readonly text?: string,
- readonly unit?: string, // if it ends with a '?', it's optional (see absmov)
+ /** If the unit ends with a '?', it is optional (e.g. absmov). */
+ readonly unit?: string,
readonly description?: string,
+ /**
+ * The arguments that need to be present if this argument is used (e.g. when a tool
+ * takes a keyword and a "parameter" for the keyword, as in "autoaim ent <entity>")
+ */
readonly children?: ToolArgument[],
- readonly otherwiseChildren?: ToolArgument[], // children if this argument didn't match (better name pls?)
+ /**
+ * This argument's children if the argument isn't used (e.g. autoaim takes either an
+ * entity or a coordinate) (better name?!)
+ */
+ readonly otherwiseChildren?: ToolArgument[],
) { }
}
export const tools: ToolDefinition = {
strafe: {
- isOrderDetermined: false,
+ hasFixedOrder: false,
hasOff: true,
durationIndex: -1,
arguments: [
@@ -62,7 +81,7 @@ export namespace TASTool {
index: 5,
},
autojump: {
- isOrderDetermined: true,
+ hasFixedOrder: true,
hasOff: true,
durationIndex: -1,
arguments: [
@@ -72,7 +91,7 @@ export namespace TASTool {
index: 3,
},
absmov: {
- isOrderDetermined: true,
+ hasFixedOrder: true,
hasOff: true,
durationIndex: -1,
arguments: [
@@ -83,7 +102,7 @@ export namespace TASTool {
index: 4,
},
setang: {
- isOrderDetermined: true,
+ hasFixedOrder: true,
hasOff: false,
durationIndex: 3,
arguments: [
@@ -96,7 +115,7 @@ export namespace TASTool {
index: 1,
},
autoaim: {
- isOrderDetermined: true,
+ hasFixedOrder: true,
hasOff: true,
durationIndex: 3,
arguments: [
@@ -115,7 +134,7 @@ export namespace TASTool {
index: 2,
},
decel: {
- isOrderDetermined: true,
+ hasFixedOrder: true,
hasOff: true,
durationIndex: -1,
arguments: [
@@ -125,7 +144,7 @@ export namespace TASTool {
index: 6,
},
check: {
- isOrderDetermined: true,
+ hasFixedOrder: true,
hasOff: false,
durationIndex: 100, // janky hack to make this never show as an active tool
arguments: [
diff --git a/server/src/tas-script/tokenizer.ts b/server/src/tas-script/tokenizer.ts
index 58b4e45..1e2d728 100644
--- a/server/src/tas-script/tokenizer.ts
+++ b/server/src/tas-script/tokenizer.ts
@@ -1,3 +1,6 @@
+//! Tokenizer, capable of tokenizing TAS scripts. It also removes comments from tokenized scripts,
+//! meaning they don't appear in the output at all.
+
import { DiagnosticCollector } from "./diagnosticCollector";
export enum TokenType {
@@ -9,13 +12,21 @@ export enum TokenType {
export class Token {
constructor(
public type: TokenType,
+ /** The raw text of the token. */
public text: string,
public line: number,
+ /** Start column. */
public start: number,
+ /** End column. */
public end: number,
) { }
}
+/**
+ * Tokenizes a file given by `fileText` and returns the tokens and lines of the file.
+ * The tokens are returned as a two-dimensional array, where the first dimension is the line in
+ * the file the token is in and the second-dimension containing the tokens themselves.
+ */
export function tokenize(fileText: string): [Token[][], string[]] {
var tokens: Token[][] = [];
@@ -30,6 +41,10 @@ export function tokenize(fileText: string): [Token[][], string[]] {
return [tokens, lines];
}
+/**
+ * Removes all tokens in comments (single line and multiline). This will also emit diagnostics
+ * for any unclosed or unopened multiline comments.
+ */
function removeComments(tokens: Token[][]) {
var lineIndex = 0;
var tokenIndex = 0;
@@ -103,6 +118,7 @@ namespace Tokenizer {
var lineNumber: number = 0;
var text: string = "";
+ /** Tokenizes a single line of a script and returns the tokens of that line. */
export function tokenizeLine(lineText: string, ln: number): Token[] {
index = 0;
lineNumber = ln;
@@ -123,6 +139,7 @@ namespace Tokenizer {
}
}
+ /** Advances `index` past the next token and return a result, consisting of either an EOF or a `Token`. */
function next(): Result | undefined {
if (index >= text.length) return new Result(ResultType.End);
@@ -135,7 +152,18 @@ namespace Tokenizer {
return new Result(ResultType.Token, new Token(nextType, tokenText, lineNumber, start, end));
}
- function accept(predicate: (str: string) => boolean): boolean {
+ /**
+ * Function predicate to determine whether a character should be accepted. Note that
+ * JavaScript doesn't have "chars", so this is passed a string of length 1.
+ */
+ type Predicate = (str: string) => boolean;
+
+ /**
+ * Takes a `predicate` and returns whether it accepts the current character in `text`. If it does,
+ * it advances the character index, meaning consecutive calls to `accept()` will consume more and
+ * more of the tokenizer's input string.
+ */
+ function accept(predicate: Predicate): boolean {
if (index >= text.length) return false;
const c = text[index];
@@ -146,8 +174,7 @@ namespace Tokenizer {
return false;
}
- type Predicate = (str: string) => boolean;
-
+ /** Return a `Predicate` that accepts any character in `chars`. */
function anyOf(chars: string): Predicate {
return (str: string): boolean => chars.indexOf(str) !== -1;
}
@@ -156,6 +183,7 @@ namespace Tokenizer {
const numberPredicate = anyOf("-0123456789");
const letterPredicate = anyOf("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-0123456789:().\"\'=@[]");
+ /** Advance `index` past the next token and return the corresponding `TokenType`. */
function nextTokenType(): TokenType {
// Skip whitespace
if (accept(whitespacePredicate)) {
From ec29bdfc284547c1b862f64a739431f5698a9303 Mon Sep 17 00:00:00 2001
From: david072 <david.g.ganz@gmail.com>
Date: Thu, 19 Dec 2024 16:01:47 +0100
Subject: [PATCH 2/2] Improve handling of path arguments
The `start` and `rngmanip` statements both (might) take a path argument,
which needs to be handled in a special way due to how the script is
tokenized. We now have a unified method in the parser for accepting such
strings.
The method `acceptConsecutiveTokens()` will accept any number of tokens
that are adjacent to each other (i.e. the first token's end is the same
as the next token's start, due to the end being exclusive) and return
how many tokens were accepted. This makes it possible to accept
continuous strings (like e.g. paths), but reject anything separated by a
space.
---
server/src/tas-script/tasScript.ts | 70 ++++++++++++++++++------------
1 file changed, 42 insertions(+), 28 deletions(-)
diff --git a/server/src/tas-script/tasScript.ts b/server/src/tas-script/tasScript.ts
index ec66d91..7970b58 100644
--- a/server/src/tas-script/tasScript.ts
+++ b/server/src/tas-script/tasScript.ts
@@ -90,31 +90,13 @@ export class TASScript {
this.expectText("Expected start", "start");
const startType = this.expectText("Expected start type", "map", "save", "cm", "now", "next");
- const checkStartTypeArgument = (startType: string, isNested: boolean) => {
+ /** Accepts the path after map/save/cm start types. */
+ const acceptStartPathArgument = (startType: string) => {
if (startType !== "map" && startType !== "save" && startType !== "cm") return false;
-
- var i = this.tokenIndex;
- var tokenCount = 2;
- var token1 = this.tokens[this.lineIndex][i];
- var token2 = this.tokens[this.lineIndex][i + 1];
- if (token1 === undefined && token2 === undefined) {
- this.expectText("Expected parameter");
- return;
- }
-
- // Accept tokens until there are no more, or two tokens are not adjacent
- // E.g.: `1/2/3` - Parsed as separate tokens, but adjacent -> accepted
- // `1/2 3` - First three tokens accepted (^), fourth not adjacent -> error on fourth
- while (true) {
- if (token2 === undefined) return;
- if (token1.end !== token2.start) {
- this.expectCount("Ignored parameters", tokenCount + 1 + (isNested ? 1 : 0));
- }
-
- i++;
- token1 = this.tokens[this.lineIndex][i];
- token2 = this.tokens[this.lineIndex][i + 1];
- tokenCount++;
+ const count = this.acceptConsecutiveTokens();
+ if (count === 0) {
+ const lastToken = this.tokens[this.lineIndex][this.tokenIndex - 1];
+ DiagnosticCollector.addDiagnosticToLine(lastToken.line, lastToken.end, "Expected arguments");
}
};
@@ -127,14 +109,14 @@ export class TASScript {
if (startType === "now")
this.expectCount("Ignored parameters", 3);
else
- checkStartTypeArgument(startType, true);
+ acceptStartPathArgument(startType);
}
}
else {
if (startType === "now")
this.expectCount("Ignored parameters", 2);
else
- checkStartTypeArgument(startType, false);
+ acceptStartPathArgument(startType);
}
}
@@ -147,8 +129,11 @@ export class TASScript {
if (token === undefined) break;
if (token.type === TokenType.String && token.text === "rngmanip") {
- this.expectText("Expected parameter");
- this.expectCount("Ignored parameters", 2)
+ const count = this.acceptConsecutiveTokens();
+ if (count === 0) {
+ const lastToken = this.tokens[this.lineIndex][this.tokenIndex - 1];
+ DiagnosticCollector.addDiagnosticToLine(lastToken.line, lastToken.end, "Expected arguments");
+ }
} else {
this.lineIndex--;
}
@@ -315,6 +300,35 @@ export class TASScript {
return DiagnosticCollector.getDiagnostics();
}
+ /**
+ * Accepts any number of consecutive tokens in the current line and returns how many were accepted.
+ *
+ * Example:
+ * `1/2/3` - Tokenized as separate tokens, but consecutive - accepted! \
+ * `1/2 3` - First three tokens are accepted, the fourth token is not adjacent - only three tokens accepted.
+ */
+ private acceptConsecutiveTokens(): number {
+ var token1 = this.maybeNext();
+ if (token1 === undefined) return 0;
+ var token2 = this.maybeNext();
+ if (token2 === undefined) return 1;
+
+ var count = 2;
+ while (true) {
+ // Return `count - 1`, since we didn't actually accept the last token (`token2`)
+ if (token2 === undefined) return count - 1;
+ if (token1.end !== token2.start) {
+ this.tokenIndex--;
+ return count - 1;
+ }
+
+ token1 = token2;
+ token2 = this.maybeNext();
+ count++;
+ }
+ }
+
+
private parseButtonsField() {
if (this.tokens[this.lineIndex].length <= this.tokenIndex) return;
if (this.isNextType(TokenType.Pipe)) {