Add NVSwitch device ID for p5.48xlarge instance and test gpu_health_check for multi-gpu instances which require nvidia fabric manager to be enabled. (#2431)

Co-authored-by: Himani Deshpande <[email protected]>

Author: himani2411

cookbooks/aws-parallelcluster-platform/kitchen.platform-config.yml

@@ -78,12 +78,14 @@ suites:
         - 'resource:package { "package_name": "dkms" }'
         - resource:build_tools
         - recipe:aws-parallelcluster-platform::nvidia_install
+#        - resource:fabric_manager:configure # Needed for Multi-gpu instance like p5.48xlarge
       resource: gdrcopy:configure
       cluster:
         nvidia:
           enabled: true
     driver:
       instance_type: g4dn.2xlarge
+#      instance_type: p5.48xlarge
   - name: intel_hpc
     run_list:
       - recipe[aws-parallelcluster-tests::setup]

cookbooks/aws-parallelcluster-platform/resources/fabric_manager/partial/_fabric_manager_common.rb

@@ -63,8 +63,10 @@ def _nvidia_driver_version
 
 # Get number of nv switches
 def get_nvswitches
-  # NVSwitch device id is 10de:1af1
-  nvswitch_check = Mixlib::ShellOut.new("lspci -d 10de:1af1 | wc -l")
-  nvswitch_check.run_command
-  nvswitch_check.stdout.strip.to_i
+  #  A100 (P4) and H100(P5) systems have NVSwitches
+  # NVSwitch device id is 10de:1af1 for P4 instance
+  # NVSwitch device id is 10de:22a3 for P5 instance
+  nvswitch_check_p4 = shell_out("lspci -d 10de:1af1 | wc -l")
+  nvswitch_check_p5 = shell_out("lspci -d 10de:22a3 | wc -l")
+  nvswitch_check_p4.stdout.strip.to_i + nvswitch_check_p5.stdout.strip.to_i
 end

cookbooks/aws-parallelcluster-slurm/kitchen.slurm-config.yml

@@ -84,10 +84,12 @@ suites:
         - /gpu_health_check_execution/
     driver:
       instance_type: g4dn.xlarge
+#      instance_type: p5.48xlarge
     attributes:
       dependencies:
         - recipe:aws-parallelcluster-slurm::mock_slurm
         - resource:node_attributes
+#        - resource:fabric_manager:configure # Needed for Multi-gpu instance like p5.48xlarge
       cluster:
         node_type: HeadNode
         scheduler: 'slurm'