Send bitbake -e <package> output to the server for further processing

client/package.json

@@ -319,6 +319,11 @@
         "title": "BitBake: Run task for recipe",
         "description": "Run a specific task for a bitbake recipe."
       },
+      {
+        "command": "bitbake.scan-recipe-env",
+        "title": "BitBake: Scan Recipe",
+        "description": "Run 'bitbake -e' on the chosen recipe to improve the language services such as hover definition"
+      },
       {
         "command": "bitbake.drop-recipe",
         "title": "BitBake: Drop a recipe from the active workspace",
@@ -434,6 +439,10 @@
           "command": "bitbake.run-task",
           "group": "0@bitbake_recipe@2"
         },
+        {
+          "command": "bitbake.scan-recipe-env",
+          "group": "0@bitbake_recipe@3"
+        },
         {
           "command": "bitbake.devtool-modify",
           "group": "1@bitbake_devtool@0"

client/src/__tests__/unit-tests/driver/bitbake-driver.test.ts

@@ -5,6 +5,7 @@
 
 import * as fs from 'fs'
 import { BitbakeDriver } from '../../../driver/BitbakeDriver'
+import { type BitbakeTaskDefinition } from '../../../ui/BitbakeTaskProvider'
 
 describe('BitbakeDriver Tests', () => {
   it('should protect from shell injections', (done) => {
@@ -76,4 +77,24 @@ describe('BitbakeDriver Tests', () => {
     const script = driver.composeBitbakeScript('bitbake busybox')
     expect(script).toEqual(expect.stringContaining("docker run --rm -it -v /home/user/yocto:/workdir crops/poky --workdir=/workdir /bin/bash -c '. /home/user/yocto/poky/oe-init-build-env && bitbake busybox'"))
   })
+
+  describe('composeBitbakeCommand', () => {
+    let bitbakeDriver: BitbakeDriver
+    beforeEach(() => {
+      bitbakeDriver = new BitbakeDriver()
+    })
+
+    it('should compose bitbake command for scanning recipe environment', () => {
+      // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+      const bitbakeTaskDefinition = {
+        recipes: ['recipe1'],
+        options: {
+          env: true
+        }
+      } as BitbakeTaskDefinition
+
+      const command = bitbakeDriver.composeBitbakeCommand(bitbakeTaskDefinition)
+      expect(command).toEqual('bitbake recipe1 -e')
+    })
+  })
 })

client/src/driver/BitbakeDriver.ts

@@ -134,6 +134,13 @@ export class BitbakeDriver {
       return sanitizeForShell(bitbakeTaskDefinition.specialCommand) as string
     }
 
+    const OPTIONS_MAP: Record<keyof BitbakeTaskDefinition['options'], string> = {
+      continue: '-k',
+      force: '-f',
+      parseOnly: '-p',
+      env: '-e'
+    }
+
     let command = 'bitbake'
 
     bitbakeTaskDefinition.recipes?.forEach(recipe => {
@@ -142,14 +149,13 @@ export class BitbakeDriver {
     if (bitbakeTaskDefinition.task !== undefined) {
       command = appendCommandParam(command, `-c ${sanitizeForShell(bitbakeTaskDefinition.task)}`)
     }
-    if (bitbakeTaskDefinition.options?.continue === true) {
-      command = appendCommandParam(command, '-k')
-    }
-    if (bitbakeTaskDefinition.options?.force === true) {
-      command = appendCommandParam(command, '-f')
-    }
-    if (bitbakeTaskDefinition.options?.parseOnly === true) {
-      command = appendCommandParam(command, '-p')
+    const options = bitbakeTaskDefinition.options
+    if (options !== undefined) {
+      Object.keys(options).forEach(key => {
+        if (options[key as keyof BitbakeTaskDefinition['options']] === true) {
+          command = appendCommandParam(command, OPTIONS_MAP[key as keyof BitbakeTaskDefinition['options']])
+        }
+      })
     }
 
     return command

client/src/driver/BitbakeRecipeScanner.ts

@@ -0,0 +1,81 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) 2023 Savoir-faire Linux. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+
+import * as vscode from 'vscode'
+import { type LanguageClient } from 'vscode-languageclient/node'
+import { logger } from '../lib/src/utils/OutputLogger'
+import { runBitbakeTask } from '../ui/BitbakeCommands'
+import { type BitbakeCustomExecution, type BitbakeTaskProvider } from '../ui/BitbakeTaskProvider'
+import { RequestMethod, type RequestParams } from '../lib/src/types/requests'
+
+export class BitbakeRecipeScanner {
+  private _languageClient: LanguageClient | undefined
+  private _currentUriForScan: string = ''
+  private _pendingRecipeScanTasks: { task: vscode.Task, uri: string } | null = null
+
+  public scanResults: string = ''
+  public processedScanResults: Record<string, unknown> | undefined
+
+  async scan (chosenRecipe: string, taskProvider: BitbakeTaskProvider, uri: any): Promise<void> {
+    if (chosenRecipe === '') {
+      logger.debug('[BitbakeRecipeScanner] No recipe chosen for scan')
+      return
+    }
+
+    const taskName = 'Bitbake: Scan recipe env'
+    const scanRecipeEnvTask = new vscode.Task(
+      { type: 'bitbake', recipes: [chosenRecipe], options: { parseOnly: true, env: true } },
+      vscode.TaskScope.Workspace,
+      taskName,
+      'bitbake'
+    )
+
+    const runningTasks = vscode.tasks.taskExecutions
+    if (runningTasks.some((execution) => execution.task.name === taskName)) {
+      logger.debug('[BitbakeRecipeScanner] Recipe scan is already running, pushing to pending tasks')
+      this._pendingRecipeScanTasks = { task: scanRecipeEnvTask, uri }
+      return
+    }
+
+    logger.debug(`[BitbakeRecipeScanner] Scanning recipe env: ${uri}`)
+    this._currentUriForScan = uri
+
+    await runBitbakeTask(scanRecipeEnvTask, taskProvider)
+  }
+
+  subscribeToTaskEnd (context: vscode.ExtensionContext, taskProvider: BitbakeTaskProvider): void {
+    context.subscriptions.push(vscode.tasks.onDidEndTask(async (e) => {
+      if (e.execution.task.name === 'Bitbake: Scan recipe env') {
+        const executionEngine = e.execution.task.execution as BitbakeCustomExecution
+        if (executionEngine !== undefined) {
+          this.scanResults = executionEngine.pty?.outputDataString ?? ''
+          if (this._languageClient === undefined) {
+            logger.error('[onDidEndTask] Language client not set, unable to forward recipe environment to the server')
+          } else {
+            if (this.scanResults !== '') {
+              logger.debug('[onDidEndTask] Sending recipe environment to the server')
+              const requestParam: RequestParams['ProcessRecipeScanResults'] = { scanResults: this.scanResults, uri: this._currentUriForScan }
+              await this._languageClient.sendNotification(RequestMethod.ProcessRecipeScanResults, requestParam)
+            }
+          }
+        }
+
+        if (this._pendingRecipeScanTasks !== null) {
+          logger.debug(`[onDidEndTask] Running the pending recipe scan task. url: ${this._pendingRecipeScanTasks.uri}`)
+          this._currentUriForScan = this._pendingRecipeScanTasks.uri
+          await runBitbakeTask(this._pendingRecipeScanTasks.task, taskProvider)
+          this._pendingRecipeScanTasks = null
+        }
+      }
+    }))
+  }
+
+  setLanguageClient (client: LanguageClient): void {
+    this._languageClient = client
+  }
+}
+
+const bitbakeRecipeScanner = new BitbakeRecipeScanner()
+export default bitbakeRecipeScanner

client/src/extension.ts

@@ -20,6 +20,7 @@ import { BitbakeDocumentLinkProvider } from './documentLinkProvider'
 import { DevtoolWorkspacesView } from './ui/DevtoolWorkspacesView'
 import { bitbakeESDKMode, setBitbakeESDKMode } from './driver/BitbakeESDK'
 import path from 'path'
+import bitbakeRecipeScanner from './driver/BitbakeRecipeScanner'
 
 let client: LanguageClient
 const bitbakeDriver: BitbakeDriver = new BitbakeDriver()
@@ -91,6 +92,9 @@ export async function activate (context: vscode.ExtensionContext): Promise<void>
 
   taskProvider = vscode.tasks.registerTaskProvider('bitbake', bitbakeTaskProvider)
 
+  bitbakeRecipeScanner.setLanguageClient(client)
+  bitbakeRecipeScanner.subscribeToTaskEnd(context, bitbakeTaskProvider)
+
   clientNotificationManager.setMemento(context.workspaceState)
   bitbakeRecipesView = new BitbakeRecipesView(bitbakeWorkspace, bitBakeProjectScanner)
   bitbakeRecipesView.registerView(context)
@@ -113,7 +117,7 @@ export async function activate (context: vscode.ExtensionContext): Promise<void>
         event.affectsConfiguration('bitbake.pathToBitbakeFolder') ||
         event.affectsConfiguration('bitbake.pathToBuildFolder') ||
         event.affectsConfiguration('bitbake.commandWrapper')) {
-      await clientNotificationManager.resetNeverShowAgain('custom/bitbakeSettingsError')
+      await clientNotificationManager.resetNeverShowAgain('bitbake/bitbakeSettingsError')
       logger.debug('Bitbake settings changed')
       updatePythonPath()
       void vscode.commands.executeCommand('bitbake.rescan-project')

client/src/language/languageClient.ts

@@ -88,7 +88,7 @@ export async function activateLanguageServer (context: ExtensionContext): Promis
   const client: LanguageClient = new LanguageClient('bitbake', 'Bitbake Language Server', serverOptions, clientOptions)
   requestsManager.client = client
 
-  client.onRequest('custom/verifyConfigurationFileAssociation', async (param) => {
+  client.onRequest('bitbake/verifyConfigurationFileAssociation', async (param) => {
     if (param.filePath?.endsWith('.conf') === true) {
       const doc = await workspace.openTextDocument(param.filePath)
       const { languageId } = doc

client/src/lib/src/types/notifications.ts

@@ -10,7 +10,7 @@ enum NotificationType {
 }
 
 export const NotificationMethod: Record<NotificationType, string> = {
-  [NotificationType.EmbeddedLanguageDocs]: 'custom/EmbeddedLanguageDocs'
+  [NotificationType.EmbeddedLanguageDocs]: 'bitbake/EmbeddedLanguageDocs'
 }
 
 export interface NotificationParams {

client/src/lib/src/types/requests.ts

@@ -8,20 +8,24 @@ import { type EmbeddedLanguageType } from './embedded-languages'
 
 export enum RequestType {
   EmbeddedLanguageTypeOnPosition = 'EmbeddedLanguageTypeOnPosition',
-  getLinksInDocument = 'getLinksInDocument'
+  getLinksInDocument = 'getLinksInDocument',
+  ProcessRecipeScanResults = 'ProcessRecipeScanResults'
 }
 
 export const RequestMethod: Record<RequestType, string> = {
-  [RequestType.EmbeddedLanguageTypeOnPosition]: 'custom/requestEmbeddedLanguageTypeOnPosition',
-  [RequestType.getLinksInDocument]: 'custom/getLinksInDocument'
+  [RequestType.EmbeddedLanguageTypeOnPosition]: 'bitbake/requestEmbeddedLanguageDocInfos',
+  [RequestType.getLinksInDocument]: 'bitbake/getLinksInDocument',
+  [RequestType.ProcessRecipeScanResults]: 'bitbake/ProcessRecipeScanResults'
 }
 
 export interface RequestParams {
   [RequestType.EmbeddedLanguageTypeOnPosition]: { uriString: string, position: Position }
   [RequestType.getLinksInDocument]: { documentUri: string }
+  [RequestType.ProcessRecipeScanResults]: { scanResults: string, uri: any }
 }
 
 export interface RequestResult {
   [RequestType.EmbeddedLanguageTypeOnPosition]: Promise<EmbeddedLanguageType | undefined | null> // for unknown reasons, the client receives null instead of undefined
   [RequestType.getLinksInDocument]: Promise<Array<{ value: string, range: Range }>>
+  [RequestType.ProcessRecipeScanResults]: Record<string, unknown> | undefined
 }

client/src/ui/BitbakeCommands.ts

@@ -23,6 +23,7 @@ import { finishProcessExecution } from '../lib/src/utils/ProcessUtils'
 import { type SpawnSyncReturns } from 'child_process'
 import { clientNotificationManager } from './ClientNotificationManager'
 import { configureDevtoolSDKFallback } from '../driver/BitbakeESDK'
+import bitbakeRecipeScanner from '../driver/BitbakeRecipeScanner'
 
 let parsingPending = false
 let bitbakeSanity = false
@@ -31,6 +32,7 @@ export function registerBitbakeCommands (context: vscode.ExtensionContext, bitba
   context.subscriptions.push(vscode.commands.registerCommand('bitbake.parse-recipes', async () => { await parseAllrecipes(bitbakeWorkspace, bitbakeTaskProvider) }))
   context.subscriptions.push(vscode.commands.registerCommand('bitbake.build-recipe', async (uri) => { await buildRecipeCommand(bitbakeWorkspace, bitbakeTaskProvider.bitbakeDriver, uri) }))
   context.subscriptions.push(vscode.commands.registerCommand('bitbake.clean-recipe', async (uri) => { await cleanRecipeCommand(bitbakeWorkspace, bitbakeTaskProvider.bitbakeDriver, uri) }))
+  context.subscriptions.push(vscode.commands.registerCommand('bitbake.scan-recipe-env', async (uri) => { await scanRecipeCommand(bitbakeWorkspace, bitbakeTaskProvider, uri) }))
   context.subscriptions.push(vscode.commands.registerCommand('bitbake.run-task', async (uri, task) => { await runTaskCommand(bitbakeWorkspace, bitbakeTaskProvider.bitbakeDriver, uri, task) }))
   context.subscriptions.push(vscode.commands.registerCommand('bitbake.drop-recipe', async (uri) => { await dropRecipe(bitbakeWorkspace, uri) }))
   context.subscriptions.push(vscode.commands.registerCommand('bitbake.watch-recipe', async (recipe) => { await addActiveRecipe(bitbakeWorkspace, recipe) }))
@@ -45,7 +47,8 @@ export function registerBitbakeCommands (context: vscode.ExtensionContext, bitba
           void parseAllrecipes(bitbakeWorkspace, bitbakeTaskProvider)
         }
       }
-    }))
+    })
+  )
 }
 
 export function registerDevtoolCommands (context: vscode.ExtensionContext, bitbakeWorkspace: BitbakeWorkspace, bitBakeProjectScanner: BitBakeProjectScanner): void {
@@ -114,6 +117,26 @@ async function cleanRecipeCommand (bitbakeWorkspace: BitbakeWorkspace, bitbakeDr
   }
 }
 
+async function scanRecipeCommand (bitbakeWorkspace: BitbakeWorkspace, taskProvider: BitbakeTaskProvider, uri?: any): Promise<void> {
+  const chosenRecipe = await selectRecipe(bitbakeWorkspace, uri, false)
+
+  if (chosenRecipe === undefined) {
+    logger.debug('Command: scan-recipe-env: chosen recipe is undefined. Abort command')
+    return
+  }
+
+  logger.debug('Command: scan-recipe-env')
+
+  if (!bitbakeSanity && !(await taskProvider.bitbakeDriver?.checkBitbakeSettingsSanity())) {
+    logger.warn('bitbake settings are not sane, Abort scan')
+    return
+  }
+
+  bitbakeSanity = true
+
+  await bitbakeRecipeScanner.scan(chosenRecipe, taskProvider, uri)
+}
+
 async function runTaskCommand (bitbakeWorkspace: BitbakeWorkspace, bitbakeDriver: BitbakeDriver, uri?: any, task?: any): Promise<void> {
   const chosenRecipe = await selectRecipe(bitbakeWorkspace, uri)
   if (chosenRecipe !== undefined) {
@@ -195,7 +218,7 @@ async function dropRecipe (bitbakeWorkspace: BitbakeWorkspace, uri?: string): Pr
   }
 }
 
-async function runBitbakeTask (task: vscode.Task, taskProvider: vscode.TaskProvider): Promise<void> {
+export async function runBitbakeTask (task: vscode.Task, taskProvider: vscode.TaskProvider): Promise<void> {
   let resolvedTask = taskProvider.resolveTask(task, new vscode.CancellationTokenSource().token)
   if (resolvedTask instanceof Promise) {
     resolvedTask = await resolvedTask

client/src/ui/BitbakeTaskProvider.ts

@@ -15,6 +15,7 @@ export interface BitbakeTaskDefinition extends vscode.TaskDefinition {
     continue?: boolean
     force?: boolean
     parseOnly?: boolean
+    env?: boolean // Recipe environment
   }
   specialCommand?: string
 }
@@ -36,8 +37,16 @@ export class BitbakeTaskProvider implements vscode.TaskProvider {
 
   resolveTask (task: vscode.Task, token: vscode.CancellationToken): vscode.ProviderResult<vscode.Task> | undefined {
     const bitbakeTaskDefinition: BitbakeTaskDefinition = task.definition as any
-    if (bitbakeTaskDefinition.recipes?.[0] !== undefined || bitbakeTaskDefinition.options?.parseOnly === true || bitbakeTaskDefinition.specialCommand !== undefined) {
+
+    const canResolveTask = (bitbakeTaskDefinition.recipes?.[0] !== undefined ||
+    bitbakeTaskDefinition.options?.parseOnly === true ||
+    bitbakeTaskDefinition.options?.env === true ||
+    bitbakeTaskDefinition.specialCommand !== undefined)
+
+    if (canResolveTask) {
       const bitbakeCommand = this.bitbakeDriver.composeBitbakeCommand(bitbakeTaskDefinition)
+      const problemMatchers = ['$bitbake-ParseError', '$bitbake-Variable', '$bitbake-generic', '$bitbake-task-error', '$bitbake-UnableToParse']
+
       const resolvedTask = new vscode.Task(
         task.definition,
         task.scope ?? vscode.TaskScope.Workspace,
@@ -51,7 +60,7 @@ export class BitbakeTaskProvider implements vscode.TaskProvider {
             this.bitbakeDriver.composeBitbakeScript(bitbakeCommand))
           return pty
         }),
-        ['$bitbake-ParseError', '$bitbake-Variable', '$bitbake-compilation-python-function', '$bitbake-execution-error', '$bitbake-task-error', '$bitbake-UnableToParse']
+        problemMatchers
       )
       if ((bitbakeTaskDefinition.task === undefined || bitbakeTaskDefinition.task.includes('build')) &&
           bitbakeTaskDefinition.options?.parseOnly !== true) {

client/src/ui/BitbakeTerminal.ts

@@ -58,6 +58,7 @@ export class BitbakePseudoTerminal implements vscode.Pseudoterminal {
   onDidClose = this.closeEmitter.event
   onDidChangeName = this.changeNameEmitter.event
   lastExitCode: number = 0
+  outputDataString: string = ''
 
   readonly parentTerminal: BitbakeTerminal | undefined
   bitbakeDriver: BitbakeDriver
@@ -119,6 +120,9 @@ export class BitbakePseudoTerminal implements vscode.Pseudoterminal {
     processResolved.stdout?.on('data', (data) => {
       this.output(data.toString())
       logger.debug(data.toString())
+      if (this.isTaskTerminal()) {
+        this.outputDataString += data.toString()
+      }
     })
     processResolved.stdout?.once('data', () => {
       // I wanted to use appropriate events like process.on('spawn') or terminal.open() but they are not triggered at the right time for

client/src/ui/ClientNotificationManager.ts

@@ -14,7 +14,7 @@ export class ClientNotificationManager {
   }
 
   showBitbakeSettingsError (message?: string): void {
-    if (!this.checkIsNeverShowAgain('custom/bitbakeSettingsError')) {
+    if (!this.checkIsNeverShowAgain('bitbake/bitbakeSettingsError')) {
       void window.showErrorMessage(
         'BitBake could not be configured and started. To enable advanced Bitbake features, please configure the Bitbake extension.\n\n' + message,
         'Open Settings',
@@ -25,7 +25,7 @@ export class ClientNotificationManager {
           if (item === 'Open Settings') {
             void commands.executeCommand('workbench.action.openWorkspaceSettings', '@ext:yocto-project.yocto-bitbake')
           } else if (item === 'Don\'t Show Again') {
-            void this.neverShowAgain('custom/bitbakeSettingsError')
+            void this.neverShowAgain('bitbake/bitbakeSettingsError')
           }
         }, (reason) => {
           logger.warn('Could not show bitbake error dialog: ' + reason)