Feat: Add diagnostics for embedded languages

client/README.md

@@ -73,6 +73,10 @@ user settings.
 }
 ```
 
+Note that the extension deactivates the "files.trimTrailingWhitespace" setting for
+Python and Shell script, as it interferes with the functioning of features related
+to embedded languages.
+
 ## Features
 
 ### Syntax highlighting
@@ -191,6 +195,9 @@ You can also set up the SDK for the recipe by running the `Bitbake: Devtool: Con
 
 If your recipe's class is not supported, or you have an older version of poky, the `Bitbake: Devtool: Configure devtool fallback` command will add tasks to build and deploy the package through `devtool build/deploy-target`. Linting, debugging, testing and other advanced features will not be available in this mode.
 
+## Troubleshooting
+See the [TROUBLESHOOTING.md](./TROUBLESHOOTING.md) file.
+
 ## Contributing
 
 ### Reporting issues

client/TROUBLESHOOTING.md

@@ -0,0 +1,22 @@
+# Troubleshooting
+
+## Known Issues
+
+### Problems from Unknown Files Appear in the Problems Tab
+Errors and warnings appear twice in the "PROBLEMS" tab. They first appear for the BitBake files to which they belong, and then again for Bash or Python files with UUID names (ex. 9ad23ed5-9278-41e0-98cd-349750c1e2c0.py). As the first one is expected and desired, the second is not. This issue arises from the way we handle diagnostics (errors and warnings). See [Trade-offs on Diagnostics](TROUBLESHOOTING.md#trade-offs-on-diagnostics).
+
+Unfortunately, the VS Code API does not offer a way to programmatically filter the "PROBLEMS" tab. However, manual filtering is still possible. Typing `!workspaceStorage` or `!workspaceStorage/**/yocto-project.yocto-bitbake/embedded-documents` (if more precision is needed) should filter out all these unwanted problems.
+
+### Tabs from Unknown Files Open and Close Quickly
+While typing, tabs with UUID names (ex. aa2a3ba2-769c-4900-8f5f-31ea16bfbc8f.sh) might occasionally open in the tabs bar and then close shortly thereafter.  This occurs as a result of our method for handling diagnostics (errors and warnings). See [Trade-offs on Diagnostics](TROUBLESHOOTING.md#trade-offs-on-diagnostics).
+
+We haven't found a way to prevent these tabs from opening, but we try to close them as quickly as possible.
+
+## Trade-offs
+
+### Trade-offs on Diagnostics
+Some functionalities for [embedded Bash and Python code](https://code.visualstudio.com/api/language-extensions/embedded-languages) are provided by other VS Code extensions, such as ms-python.python or timonwong.shellcheck. To achieve this, we extract the Bash and Python content from the BitBake files and create temporary Bash or Python files that can be analyzed by Bash or Python extensions. We adapt the results of these extensions, and present them to the user. It works well for Completion, Definition and Hover. Unfortunately VS Code [does not yet offer such functionality for diagnostics](https://github.com/yoctoproject/vscode-bitbake/pull/18). We still achieve it by some homemade hacky technique that consists of opening the temporary documents in the background, in a way to trigger the generation of diagnostics. It brings a couple of issues, but we hope it is still worth having the diagnostics.
+
+The related issues:
+- [Problems from Unknown Files Appear in the Problems Tab](TROUBLESHOOTING.md#problems-from-unknown-files-appear-in-the-problems-tab)
+- [Tabs from Unknown Files Open and Close Quickly](TROUBLESHOOTING.md#tabs-from-unknown-files-open-and-close-quickly)

client/src/extension.ts

@@ -59,6 +59,16 @@ function updatePythonPath (): void {
   }
 }
 
+async function disableInteferingSettings (): Promise<void> {
+  const config = vscode.workspace.getConfiguration()
+  for (const languageKey of ['[python]', '[shellscript]']) {
+    const languageConfig = config.get<Record<string, unknown>>(languageKey) ?? {}
+    // 'files.trimTrailingWhitespace' modifies the embedded languages documents and breaks the mapping of the positions
+    languageConfig['files.trimTrailingWhitespace'] = false
+    await config.update(languageKey, languageConfig, vscode.ConfigurationTarget.Workspace)
+  }
+}
+
 export async function activate (context: vscode.ExtensionContext): Promise<void> {
   logger.outputChannel = vscode.window.createOutputChannel('BitBake')
 
@@ -69,6 +79,7 @@ export async function activate (context: vscode.ExtensionContext): Promise<void>
   bitbakeDriver.loadSettings(vscode.workspace.getConfiguration('bitbake'), vscode.workspace.workspaceFolders?.[0].uri.fsPath)
   const bitBakeProjectScanner: BitBakeProjectScanner = new BitBakeProjectScanner(bitbakeDriver)
   updatePythonPath()
+  await disableInteferingSettings()
   bitbakeWorkspace.loadBitbakeWorkspace(context.workspaceState)
   bitbakeTaskProvider = new BitbakeTaskProvider(bitbakeDriver)
   client = await activateLanguageServer(context)

server/src/embedded-languages/documents-manager.ts → client/src/language/EmbeddedLanguageDocsManager.ts

@@ -7,10 +7,9 @@ import { randomUUID } from 'crypto'
 import path from 'path'
 import fs from 'fs'
 
+import { type EmbeddedLanguageDoc, type EmbeddedLanguageType } from '../lib/src/types/embedded-languages'
 import { logger } from '../lib/src/utils/OutputLogger'
-
-import { type EmbeddedLanguageDocInfos, type EmbeddedLanguageType } from '../lib/src/types/embedded-languages'
-import { type EmbeddedLanguageDoc } from './utils'
+import { Range, Uri, WorkspaceEdit, workspace } from 'vscode'
 
 const EMBEDDED_DOCUMENTS_FOLDER = 'embedded-documents'
 
@@ -19,16 +18,30 @@ const fileExtensionsMap = {
   python: '.py'
 }
 
+export interface EmbeddedLanguageDocInfos {
+  uri: Uri
+  language: EmbeddedLanguageType
+  characterIndexes: number[]
+}
+
 type EmbeddedLanguageDocsRecord = Partial<Record<EmbeddedLanguageType, EmbeddedLanguageDocInfos>>
 
 export default class EmbeddedLanguageDocsManager {
   private readonly embeddedLanguageDocsInfos = new Map<string, EmbeddedLanguageDocsRecord>() // map of original uri to embedded documents infos
   private _storagePath: string | undefined
+  private readonly filesWaitingToUpdate = new Map<string, EmbeddedLanguageDoc>()
 
   get storagePath (): string | undefined {
     return this._storagePath
   }
 
+  get embeddedLanguageDocsFolder (): string | undefined {
+    if (this._storagePath === undefined) {
+      return
+    }
+    return path.join(this._storagePath, EMBEDDED_DOCUMENTS_FOLDER)
+  }
+
   async setStoragePath (newStoragePath: string | undefined): Promise<void> {
     logger.debug(`Set embedded language documents storage path. New: ${newStoragePath}. Old: ${this._storagePath}`)
     if (this._storagePath === newStoragePath) {
@@ -45,7 +58,7 @@ export default class EmbeddedLanguageDocsManager {
         const newPathToEmbeddedLanguageDocsFolder = path.join(newStoragePath, EMBEDDED_DOCUMENTS_FOLDER)
         fs.mkdir(newPathToEmbeddedLanguageDocsFolder, { recursive: true }, (err) => {
           if (err !== null) {
-            logger.error(`Failed to create embedded language documents folder: ${JSON.stringify(err)}`)
+            logger.error(`Failed to create embedded language documents folder: ${err as any}`)
           }
           resolve()
         })
@@ -58,7 +71,7 @@ export default class EmbeddedLanguageDocsManager {
         const oldPathToEmbeddedLanguageDocsFolder = path.join(this._storagePath, EMBEDDED_DOCUMENTS_FOLDER)
         fs.rmdir(oldPathToEmbeddedLanguageDocsFolder, { recursive: true }, (err) => {
           if (err !== null) {
-            logger.error(`Failed to remove embedded language documents folder: ${JSON.stringify(err)}`)
+            logger.error(`Failed to remove embedded language documents folder: ${err as any}`)
           }
           resolve()
         })
@@ -67,10 +80,14 @@ export default class EmbeddedLanguageDocsManager {
     this._storagePath = newStoragePath
   }
 
-  private registerEmbeddedLanguageDocInfos (originalUriString: string, embeddedLanguageDocInfos: EmbeddedLanguageDocInfos): void {
-    const embeddedLanguageDocs = this.embeddedLanguageDocsInfos.get(originalUriString) ?? {}
-    embeddedLanguageDocs[embeddedLanguageDocInfos.language] = embeddedLanguageDocInfos
-    this.embeddedLanguageDocsInfos.set(originalUriString, embeddedLanguageDocs)
+  private registerEmbeddedLanguageDocInfos (embeddedLanguageDoc: EmbeddedLanguageDoc, uri: Uri): void {
+    const embeddedLanguageDocInfos: EmbeddedLanguageDocInfos = {
+      ...embeddedLanguageDoc,
+      uri
+    }
+    const embeddedLanguageDocs = this.embeddedLanguageDocsInfos.get(embeddedLanguageDoc.originalUri) ?? {}
+    embeddedLanguageDocs[embeddedLanguageDoc.language] = embeddedLanguageDocInfos
+    this.embeddedLanguageDocsInfos.set(embeddedLanguageDoc.originalUri, embeddedLanguageDocs)
   }
 
   getEmbeddedLanguageDocInfos (
@@ -81,57 +98,94 @@ export default class EmbeddedLanguageDocsManager {
     return embeddedLanguageDocs?.[languageType]
   }
 
-  private getPathToEmbeddedLanguageDoc (embeddedLanguageDoc: EmbeddedLanguageDoc): string | undefined {
-    if (this.storagePath === undefined) {
+  getOriginalUri (embeddedLanguageDocUri: Uri): Uri | undefined {
+    let originalUri: Uri | undefined
+    this.embeddedLanguageDocsInfos.forEach((embeddedLanguageDocs, stringUri) => {
+      if (
+        embeddedLanguageDocs.bash?.uri.toString() === embeddedLanguageDocUri.toString() ||
+        embeddedLanguageDocs.python?.uri.toString() === embeddedLanguageDocUri.toString()
+      ) {
+        originalUri = Uri.parse(stringUri)
+      }
+    })
+    return originalUri
+  }
+
+  private createEmbeddedLanguageDocUri (embeddedLanguageDoc: EmbeddedLanguageDoc): Uri | undefined {
+    if (this.embeddedLanguageDocsFolder === undefined) {
       return undefined
     }
-    const embeddedLanguageDocInfos = this.getEmbeddedLanguageDocInfos(
-      embeddedLanguageDoc.originalUri,
-      embeddedLanguageDoc.language
-    )
-    if (embeddedLanguageDocInfos !== undefined) {
-      return embeddedLanguageDocInfos.uri.replace('file://', '')
-    }
     const randomName = randomUUID()
     const fileExtension = fileExtensionsMap[embeddedLanguageDoc.language]
     const embeddedLanguageDocFilename = randomName + fileExtension
-    const pathToEmbeddedLanguageDocsFolder = path.join(this.storagePath, EMBEDDED_DOCUMENTS_FOLDER)
-    return `${pathToEmbeddedLanguageDocsFolder}/${embeddedLanguageDocFilename}`
+    const pathToEmbeddedLanguageDocsFolder = this.embeddedLanguageDocsFolder
+    return Uri.parse(`file://${pathToEmbeddedLanguageDocsFolder}/${embeddedLanguageDocFilename}`)
+  }
+
+  async saveEmbeddedLanguageDocs (
+    embeddedLanguageDocs: EmbeddedLanguageDoc[]
+  ): Promise<void> {
+    await Promise.all(embeddedLanguageDocs.map(async (embeddedLanguageDoc) => {
+      await this.saveEmbeddedLanguageDoc(embeddedLanguageDoc)
+    }))
+  }
+
+  private async updateEmbeddedLanguageDocFile (embeddedLanguageDoc: EmbeddedLanguageDoc, uri: Uri): Promise<void> {
+    const document = await workspace.openTextDocument(uri)
+    if (document.isDirty) {
+      this.filesWaitingToUpdate.set(uri.toString(), embeddedLanguageDoc)
+      return
+    }
+    const fullRange = new Range(
+      document.positionAt(0),
+      document.positionAt(document.getText().length)
+    )
+    const workspaceEdit = new WorkspaceEdit()
+    workspaceEdit.replace(uri, fullRange, embeddedLanguageDoc.content)
+    await workspace.applyEdit(workspaceEdit)
+    await document.save()
+    this.registerEmbeddedLanguageDocInfos(embeddedLanguageDoc, uri)
+    const fileWaitingToUpdate = this.filesWaitingToUpdate.get(uri.toString())
+    if (fileWaitingToUpdate !== undefined) {
+      this.filesWaitingToUpdate.delete(uri.toString())
+      await this.updateEmbeddedLanguageDocFile(fileWaitingToUpdate, uri)
+    }
+  }
+
+  private async createEmbeddedLanguageDocFile (embeddedLanguageDoc: EmbeddedLanguageDoc): Promise<void> {
+    const uri = this.createEmbeddedLanguageDocUri(embeddedLanguageDoc)
+    if (uri === undefined) {
+      return undefined
+    }
+    try {
+      await workspace.fs.writeFile(uri, Buffer.from(embeddedLanguageDoc.content))
+      await workspace.openTextDocument(uri)
+    } catch (err) {
+      logger.error(`Failed to create embedded document: ${err as any}`)
+    }
+    this.registerEmbeddedLanguageDocInfos(embeddedLanguageDoc, uri)
   }
 
   async saveEmbeddedLanguageDoc (
     embeddedLanguageDoc: EmbeddedLanguageDoc
   ): Promise<void> {
     logger.debug(`Save embedded document (${embeddedLanguageDoc.language}) for ${embeddedLanguageDoc.originalUri}`)
-    const pathToEmbeddedLanguageDoc = this.getPathToEmbeddedLanguageDoc(embeddedLanguageDoc)
-    if (pathToEmbeddedLanguageDoc === undefined) {
-      return
+    const embeddedLanguageDocInfos = this.getEmbeddedLanguageDocInfos(
+      embeddedLanguageDoc.originalUri,
+      embeddedLanguageDoc.language
+    )
+    if (embeddedLanguageDocInfos !== undefined) {
+      await this.updateEmbeddedLanguageDocFile(embeddedLanguageDoc, embeddedLanguageDocInfos.uri)
+    } else {
+      await this.createEmbeddedLanguageDocFile(embeddedLanguageDoc)
     }
-    await new Promise<void>((resolve, reject) => {
-      fs.writeFile(pathToEmbeddedLanguageDoc, embeddedLanguageDoc.content, (err) => {
-        err !== null ? reject(err) : resolve()
-      })
-    }).then(() => {
-      const embeddedLanguageDocInfos: EmbeddedLanguageDocInfos = {
-        ...embeddedLanguageDoc,
-        uri: `file://${pathToEmbeddedLanguageDoc}`
-      }
-      this.registerEmbeddedLanguageDocInfos(embeddedLanguageDoc.originalUri, embeddedLanguageDocInfos)
-    }).catch((err) => {
-      logger.error(`Failed to create embedded document: ${err}`)
-    })
   }
 
   async deleteEmbeddedLanguageDocs (originalUriString: string): Promise<void> {
     logger.debug(`Delete embedded documents for ${originalUriString}`)
     const embeddedLanguageDocs = this.embeddedLanguageDocsInfos.get(originalUriString) ?? {}
     await Promise.all(Object.values(embeddedLanguageDocs).map(async ({ uri }) => {
-      await new Promise<void>((resolve, reject) => {
-        const pathToEmbeddedLanguageDoc = uri.replace('file://', '')
-        fs.unlink(pathToEmbeddedLanguageDoc, (err) => {
-          err !== null ? reject(err) : resolve()
-        })
-      })
+      await workspace.fs.delete(uri)
     })).then(() => {
       this.embeddedLanguageDocsInfos.delete(originalUriString)
     }).catch((err) => {

client/src/language/RequestManager.ts

@@ -11,12 +11,12 @@ import { RequestMethod, type RequestParams, type RequestResult } from '../lib/sr
 export class RequestManager {
   client: LanguageClient | undefined
 
-  getEmbeddedLanguageDocInfos = async (
+  getEmbeddedLanguageTypeOnPosition = async (
     uriString: string,
     position: Position
-  ): RequestResult['EmbeddedLanguageDocInfos'] => {
-    const params: RequestParams['EmbeddedLanguageDocInfos'] = { uriString, position }
-    return await this.client?.sendRequest(RequestMethod.EmbeddedLanguageDocInfos, params)
+  ): RequestResult['EmbeddedLanguageTypeOnPosition'] => {
+    const params: RequestParams['EmbeddedLanguageTypeOnPosition'] = { uriString, position }
+    return await this.client?.sendRequest(RequestMethod.EmbeddedLanguageTypeOnPosition, params)
   }
 }
 

client/src/language/diagnosticsSupport.ts

@@ -0,0 +1,67 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) 2023 Savoir-faire Linux. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+
+import * as vscode from 'vscode'
+
+import { getOriginalDocRange } from './utils'
+import { embeddedLanguageDocsManager } from './EmbeddedLanguageDocsManager'
+import { type EmbeddedLanguageType } from '../lib/src/types/embedded-languages'
+
+const diagnosticCollections = {
+  bash: vscode.languages.createDiagnosticCollection('bitbake-bash'),
+  python: vscode.languages.createDiagnosticCollection('bitbake-python')
+}
+
+export const updateDiagnostics = async (uri: vscode.Uri): Promise<void> => {
+  if (!uri.path.endsWith('.py') && !uri.path.endsWith('.sh')) {
+    return
+  }
+  const originalUri = embeddedLanguageDocsManager.getOriginalUri(uri)
+  if (originalUri === undefined) {
+    return
+  }
+  const originalTextDocument = await vscode.workspace.openTextDocument(originalUri)
+  await Promise.all([
+    setEmbeddedLanguageDocDiagnostics(originalTextDocument, 'bash'),
+    setEmbeddedLanguageDocDiagnostics(originalTextDocument, 'python')
+  ])
+}
+
+const setEmbeddedLanguageDocDiagnostics = async (
+  originalTextDocument: vscode.TextDocument,
+  embeddedLanguageType: EmbeddedLanguageType
+): Promise<void> => {
+  const embeddedLanguageDocInfos = embeddedLanguageDocsManager.getEmbeddedLanguageDocInfos(
+    originalTextDocument.uri.toString(),
+    embeddedLanguageType
+  )
+  if (embeddedLanguageDocInfos?.uri === undefined) {
+    return
+  }
+  const embeddedLanguageDoc = await vscode.workspace.openTextDocument(embeddedLanguageDocInfos.uri.fsPath)
+  const dirtyDiagnostics = vscode.languages.getDiagnostics(embeddedLanguageDocInfos.uri)
+  const cleanDiagnostics: vscode.Diagnostic[] = []
+  dirtyDiagnostics.forEach((diagnostic) => {
+    if (diagnostic.range === undefined) {
+      cleanDiagnostics.push(diagnostic)
+    }
+    const newRange = getOriginalDocRange(
+      originalTextDocument,
+      embeddedLanguageDoc,
+      embeddedLanguageDocInfos.characterIndexes,
+      diagnostic.range
+    )
+    if (newRange === undefined) {
+      return
+    }
+    const newDiagnostic = {
+      ...diagnostic,
+      range: newRange
+    }
+    cleanDiagnostics.push(newDiagnostic)
+  })
+  const diagnosticCollection = diagnosticCollections[embeddedLanguageType]
+  diagnosticCollection.set(originalTextDocument.uri, cleanDiagnostics)
+}