[NativeAOT-LLVM] Add linux-arm64 build and testing

eng/liveBuilds.targets

@@ -32,7 +32,8 @@
          does not scale for local development because it requires you to build the same host tools for each target configuration.
          E. g. you need to build the same Debug compiler 3 times if you want to compile code in Debug, Checked and Release. -->
     <IlcConfig Condition="'$(IlcConfig)' == ''">$(CoreCLRConfiguration)</IlcConfig>
-    <CoreCLRCrossILCompilerDir Condition="'$(CoreCLRCrossILCompilerDir)' != '' and '$(TargetsWasm)' == 'true'">$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'coreclr', '$(HostOS).$(BuildArchitecture).$(IlcConfig)', 'ilc'))</CoreCLRCrossILCompilerDir>
+    <IlcHostArch Condition="'$(IlcHostArch)' == ''">$(BuildArchitecture)</IlcHostArch>
+    <CoreCLRCrossILCompilerDir Condition="'$(CoreCLRCrossILCompilerDir)' != '' and '$(TargetsWasm)' == 'true'">$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'coreclr', '$(HostOS).$(IlcHostArch).$(IlcConfig)', 'ilc'))</CoreCLRCrossILCompilerDir>
 
     <Crossgen2Dir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'crossgen2-published'))</Crossgen2Dir>
     <Crossgen2InBuildDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', '$(BuildArchitecture)', 'crossgen2'))</Crossgen2InBuildDir>

eng/native/gen-buildsys.sh

@@ -97,6 +97,7 @@ fi
 cmake_extra_defines_wasm=()
 if [[ "$host_arch" == "wasm" ]]; then
     if [[ "$target_os" == "browser" ]]; then
+        export EMSDK_QUIET=1 && source $EMSDK/emsdk_env.sh
         cmake_command="emcmake $cmake_command"
     elif [[ "$target_os" == "wasi" ]]; then
         if [[ -z $WASI_SDK_PATH ]]; then

eng/pipelines/common/global-build-job.yml

@@ -207,19 +207,19 @@ jobs:
       - ${{ if ne(parameters.hostedOs, 'windows') }}:
         - script: /tmp/docker exec -t -u 0 naot-llvm-ci-container sh -c "tdnf install -y sudo"
           displayName: Install sudo
-        - script: $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-build-and-test-tools.sh $(Build.SourcesDirectory)/wasm-tools
+        - script: $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-build-and-test-tools.sh
           displayName: Install NativeAOT-LLVM build and test tools
       - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-emscripten.ps1 $(Build.SourcesDirectory)/wasm-tools -CI
         displayName: Install/activate emscripten
       - ${{ if eq(parameters.hostedOs, 'windows') }}:
         - script: call $(Build.SourcesDirectory)/eng/pipelines/runtimelab/set-cmake-path.cmd
           displayName: Set CMake path
-      - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-llvm.ps1 -CloneDir $(Build.SourcesDirectory)/wasm-tools -Configs ${{ parameters.buildConfig }} -CI
+      - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-llvm.ps1 -CloneDir $(Build.SourcesDirectory)/wasm-tools -Configs $(_BuildConfig) -Arch $(hostedTargetArch) -CI
         displayName: Install/build LLVM
       - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-nodejs.ps1 $(Build.SourcesDirectory)/wasm-tools
         displayName: Install NodeJS
 
-    - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), or(eq(parameters.platform, 'wasi_wasm_win'), eq(parameters.platform, 'wasi_wasm_linux_x64_naot_llvm'))) }}:
+    - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.osGroup, 'wasi')) }}:
       # Install Wasi Wasm dependencies: wasi-sdk, wasmtime
       - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-wasi-sdk.ps1 -CI -InstallDir $(Build.SourcesDirectory)/wasm-tools
         displayName: Install wasi-sdk
@@ -228,7 +228,7 @@ jobs:
       - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-jco.ps1 $(Build.SourcesDirectory)
         displayName: Install Jco
 
-    - ${{ if or(eq(parameters.platform, 'browser_wasm_win'), and(eq(parameters.platform, 'wasi_wasm_win'), not(eq(parameters.runtimeFlavor, 'coreclr')))) }}:
+    - ${{ if and(eq(parameters.hostedOs, 'windows'), eq(parameters.archType, 'wasm')) }}:
       # Update machine certs
       - task: PowerShell@2
         displayName: Update machine certs

eng/pipelines/common/platform-matrix.yml

@@ -500,10 +500,10 @@ jobs:
         buildConfig: ${{ parameters.buildConfig }}
         ${{ insert }}: ${{ parameters.jobParameters }}
 
-# Browser WebAssembly Linux X64 for NAOT-LLVM
+# WebAssembly Linux for NAOT-LLVM
 # Use a different name to differentiate from upstream as we need the -sanitizer image to get build tools that are not present in the stock browser_wasm image.
 
-- ${{ if containsValue(parameters.platforms, 'Browser_wasm_linux_x64_naot_llvm') }}:
+- ${{ if containsValue(parameters.platforms, 'browser_wasm_linux_x64_naot_llvm') }}:
   - template: xplat-setup.yml
     parameters:
       jobTemplate: ${{ parameters.jobTemplate }}
@@ -512,19 +512,16 @@ jobs:
       osGroup: browser
       archType: wasm
       targetRid: browser-wasm
-      platform: Browser_wasm_linux_x64_naot_llvm
+      platform: browser_wasm_linux_x64_naot_llvm
       shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
-      container: linux_x64_sanitizer_naot_llvm
+      container: linux_x64_naot_llvm
       jobParameters:
         hostedOs: linux
         runtimeFlavor: ${{ parameters.runtimeFlavor }}
         stagedBuild: ${{ parameters.stagedBuild }}
         buildConfig: ${{ parameters.buildConfig }}
         ${{ insert }}: ${{ parameters.jobParameters }}
 
-# WASI WebAssembly Linux X64 for NAOT-LLVM
-# Use a different name to differentiate from upstream as we need the -sanitizer image to get build tools that are not present in the stock wasi-wasm image.
-
 - ${{ if containsValue(parameters.platforms, 'wasi_wasm_linux_x64_naot_llvm') }}:
   - template: xplat-setup.yml
     parameters:
@@ -536,9 +533,30 @@ jobs:
       targetRid: wasi-wasm
       platform: wasi_wasm_linux_x64_naot_llvm
       shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
-      container: linux_x64_sanitizer_naot_llvm
+      container: linux_x64_naot_llvm
+      jobParameters:
+        hostedOs: linux
+        runtimeFlavor: ${{ parameters.runtimeFlavor }}
+        stagedBuild: ${{ parameters.stagedBuild }}
+        buildConfig: ${{ parameters.buildConfig }}
+        ${{ insert }}: ${{ parameters.jobParameters }}
+
+- ${{ if containsValue(parameters.platforms, 'browser_wasm_linux_arm64_naot_llvm') }}:
+  - template: xplat-setup.yml
+    parameters:
+      jobTemplate: ${{ parameters.jobTemplate }}
+      helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
+      variables: ${{ parameters.variables }}
+      osGroup: browser
+      archType: wasm
+      targetRid: browser-wasm
+      platform: browser_wasm_linux_arm64_naot_llvm
+      shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
+      container: linux_arm64_naot_llvm
       jobParameters:
         hostedOs: linux
+        hostedTargetArch: arm64
+        nameSuffix: arm64
         runtimeFlavor: ${{ parameters.runtimeFlavor }}
         stagedBuild: ${{ parameters.stagedBuild }}
         buildConfig: ${{ parameters.buildConfig }}

eng/pipelines/common/templates/pipeline-with-resources.yml

@@ -82,12 +82,19 @@ extends:
       # We need to be able to use 'tdnf install'. We can't because of https://github.com/dotnet/dotnet-docker/issues/788.
       # This is the hackaround: https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-489692810.
       # TODO-LLVM-Upstream: replace with an image that has our prerequisites (LLDB) already installed.
-      linux_x64_sanitizer_naot_llvm:
-        image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-amd64-sanitizer
+      linux_x64_naot_llvm:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net10.0-cross-amd64-sanitizer
         options: '--name naot-llvm-ci-container -v /usr/bin/docker:/tmp/docker:ro'
         env:
           ROOTFS_DIR: /crossrootfs/x64
 
+      # For arm64, we need to install QEMU.
+      linux_arm64_naot_llvm:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net10.0-cross-arm64
+        options: '--name naot-llvm-ci-container -v /usr/bin/docker:/tmp/docker:ro'
+        env:
+          ROOTFS_DIR: /crossrootfs/arm64
+
       # We use a CentOS Stream 8 image here to test building from source on CentOS Stream 9.
       SourceBuild_centos_x64:
         image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9

eng/pipelines/common/xplat-setup.yml

@@ -52,6 +52,12 @@ jobs:
       - name: osSubgroup
         value: ${{ parameters.osSubgroup }}
 
+      - name: hostedTargetArch
+        ${{ if ne(parameters.jobParameters.hostedTargetArch, '') }}:
+          value: ${{ parameters.jobParameters.hostedTargetArch }}
+        ${{ else }}:
+          value: x64
+
       - name: _runSmokeTestsOnlyArg
         value: '/p:RunSmokeTestsOnly=$(isRunSmokeTestsOnly)'
 

eng/pipelines/runtimelab-official.yml

@@ -38,7 +38,7 @@ extends:
           platforms:
           - browser_wasm_win
           - wasi_wasm_win
-          - Browser_wasm_linux_x64_naot_llvm
+          - browser_wasm_linux_x64_naot_llvm
           jobParameters:
             templatePath: 'templates-official'
             timeoutInMinutes: 300

eng/pipelines/runtimelab.yml

@@ -61,7 +61,7 @@ extends:
             - windows_x64
             - browser_wasm_win
             - wasi_wasm_win
-            - Browser_wasm_linux_x64_naot_llvm
+            - browser_wasm_linux_x64_naot_llvm
             - wasi_wasm_linux_x64_naot_llvm
             jobParameters:
               timeoutInMinutes: 300
@@ -101,10 +101,11 @@ extends:
             - linux_x64
             - osx_x64
             - windows_x64
-            - Browser_wasm_win
+            - browser_wasm_win
             - wasi_wasm_win
+            - browser_wasm_linux_x64_naot_llvm
             - wasi_wasm_linux_x64_naot_llvm
-            - Browser_wasm_linux_x64_naot_llvm
+            - browser_wasm_linux_arm64_naot_llvm
             jobParameters:
               timeoutInMinutes: 300
               buildArgs: -s clr.aot+libs -c $(_BuildConfig)

eng/pipelines/runtimelab/install-build-and-test-tools.sh

@@ -1,5 +1,10 @@
 #!/usr/bin/env bash
 set -e
-dir=$(dirname "$0")
-echo; $dir/install-python.sh
-echo; $dir/install-lldb.sh
+
+echo Setting EMSDK_PYTHON to /usr/bin/python3
+echo '##vso[task.setvariable variable=EMSDK_PYTHON]'/usr/bin/python3
+echo
+
+echo Installing LLDB, QEMU
+sudo tdnf install -y lldb python3-lldb qemu-user
+echo

eng/pipelines/runtimelab/install-llvm.ps1

@@ -3,11 +3,11 @@ param(
     [string]$CloneDir = $null,
     [ValidateSet("Debug","Release","Checked")][string[]]$Configs = @("Debug","Release"),
     [string]$Path = "llvm-project",
+    [string]$Arch = $null,
     [switch]$CI,
     [switch]$NoClone,
     [switch]$NoBuild
 )
-
 $ErrorActionPreference="Stop"
 
 if (!(gcm git -ErrorAction SilentlyContinue))
@@ -25,6 +25,11 @@ if ($CloneDir)
     Set-Location -Path $CloneDir
 }
 
+if (!$Arch)
+{
+    $Arch = [Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant()
+}
+
 $LlvmProjectTag = "llvmorg-18.1.3"
 if ($NoClone)
 {
@@ -47,22 +52,6 @@ else
     git clone https://github.com/llvm/llvm-project $Path --branch $LlvmProjectTag $DepthOption
 }
 
-# Set the compiler for CI on non-Windows
-if (!$IsWindows) {
-    $RepoDir = Split-path $PSScriptRoot | Split-Path | Split-Path
-
-    bash -c "build_arch=amd64 compiler=clang source $RepoDir/eng/common/native/init-compiler.sh && set | grep -e CC -e CXX -e LDFLAGS" |
-      ForEach-Object {
-        if ($CI)
-        {
-            # Split the "<name>=<value>" line into the variable's name and value.
-            $name, $value = $_ -split '=', 2
-            # Define it as a process-level environment variable in PowerShell.
-            Set-Content ENV:$name $value
-        }
-     }
-}
-
 # There is no [C/c]hecked LLVM config, so change to Debug
 foreach ($Config in $Configs | % { if ($_ -eq "Checked") { "Debug" } else { $_ } } | Select-Object -Unique)
 {
@@ -90,7 +79,23 @@ foreach ($Config in $Configs | % { if ($_ -eq "Checked") { "Debug" } else { $_ }
     }
     elseif ($env:ROOTFS_DIR)
     {
-        $CmakeConfigureCommandLine += "-DCMAKE_SYSROOT=$env:ROOTFS_DIR", "-DCMAKE_INSTALL_PREFIX=/usr/local/llvm-cross"
+        # This logic was copied from gen-buildsys.sh. Maybe we should just call it here directly?
+        $RepoRoot = Split-Path $PSScriptRoot | Split-Path | Split-Path
+        # We're assuming here that the sysroot is coming from our CI docker images.
+        $TargetTriple = (Get-ChildItem $env:ROOTFS_DIR/usr/lib/gcc).Name
+
+        bash -c "build_arch=$Arch compiler=clang source $RepoRoot/eng/common/native/init-compiler.sh && set | grep -e CC -e CXX -e LDFLAGS" |
+            ForEach-Object {
+                # Split the "<name>=<value>" line into the variable's name and value.
+                $name, $value = $_ -Split '=', 2
+                # Define it as a process-level environment variable in PowerShell.
+                Set-Content ENV:$name $value
+            }
+        $env:TARGET_BUILD_ARCH = $Arch # Used by toolchain.cmake
+        $CmakeConfigureCommandLine +=
+            "-DCMAKE_TOOLCHAIN_FILE=$RepoRoot/eng/common/cross/toolchain.cmake",
+            "-DLLVM_HOST_TRIPLE=$TargetTriple",
+            "-DCMAKE_ASM_COMPILER_VERSION=18.1.2" # Workaround for https://gitlab.kitware.com/cmake/cmake/-/issues/22995.
     }
 
     Write-Host "Invoking CMake configure: 'cmake $CmakeConfigureCommandLine'"

eng/pipelines/runtimelab/install-nodejs.ps1

@@ -1,19 +1,15 @@
 $InstallPath = $Args[0]
 $NodeJSVersion = "v20.2.0"
 
-if (!(Test-Path variable:global:IsWindows))
-{
-    $IsWindows = [Environment]::OSVersion.Platform -eq [PlatformID]::Win32NT
-}
-
-if ($IsWIndows)
+$HostArch = [Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant()
+if ($IsWindows)
 {
-    $NodeJSInstallName = "node-$NodeJSVersion-win-x64"
+    $NodeJSInstallName = "node-$NodeJSVersion-win-$HostArch"
     $NodeJSZipName = "$NodeJSInstallName.zip"
 }
 else
 {
-    $NodeJSInstallName = "node-$NodeJSVersion-linux-x64"
+    $NodeJSInstallName = "node-$NodeJSVersion-linux-$HostArch"
     $NodeJSZipName = "$NodeJSInstallName.tar.xz"
 }