fix: improve graph request list performance

update graph requests for all tenants / large queries to split data across properties/rows
KelvinTegelaar · Feb 26, 2025 · e82b27d · e82b27d
1 parent 05165f0
commit e82b27d
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 19 deletions.
diff --git a/...PPCore/Public/Entrypoints/Activity Triggers/Graph Requests/Push-ListGraphRequestQueue.ps1 b/...PPCore/Public/Entrypoints/Activity Triggers/Graph Requests/Push-ListGraphRequestQueue.ps1
@@ -21,9 +21,9 @@ function Push-ListGraphRequestQueue {
         Write-Information "Queue Table: $TableName"
         $Table = Get-CIPPTable -TableName $TableName
 
-        $Filter = "PartitionKey eq '{0}' and Tenant eq '{1}'" -f $PartitionKey, $Item.TenantFilter
+        $Filter = "PartitionKey eq '{0}' and (RowKey eq '{1}' or OriginalEntityId eq '{1}')" -f $PartitionKey, $Item.TenantFilter
         Write-Information "Filter: $Filter"
-        $Existing = Get-CIPPAzDataTableEntity @Table -Filter $Filter -Property PartitionKey, RowKey
+        $Existing = Get-CIPPAzDataTableEntity @Table -Filter $Filter -Property PartitionKey, RowKey, OriginalEntityId
         if ($Existing) {
             $null = Remove-AzDataTableEntity -Force @Table -Entity $Existing
         }
@@ -41,26 +41,25 @@ function Push-ListGraphRequestQueue {
         $RawGraphRequest = try {
             Get-GraphRequestList @GraphRequestParams
         } catch {
+            $CippException = Get-CippException -Exception $_.Exception
             [PSCustomObject]@{
                 Tenant     = $Item.TenantFilter
-                CippStatus = "Could not connect to tenant. $($_.Exception.message)"
+                CippStatus = "Could not connect to tenant. $($CippException.NormalizedMessage)"
+                CippException = [string]($CippException | ConvertTo-Json -Depth 10 -Compress)
             }
         }
-        $GraphResults = foreach ($Request in $RawGraphRequest) {
-            $Json = ConvertTo-Json -Depth 10 -Compress -InputObject $Request
-            $RowKey = $Request.id ?? (New-Guid).Guid
-            [PSCustomObject]@{
-                Tenant       = [string]$Item.TenantFilter
-                QueueId      = [string]$Item.QueueId
-                QueueType    = [string]$Item.QueueType
-                RowKey       = [string]$RowKey
-                PartitionKey = [string]$PartitionKey
-                Data         = [string]$Json
-            }
+        $Json = ConvertTo-Json -Depth 10 -Compress -InputObject $RawGraphRequest
+        $GraphResults = [PSCustomObject]@{
+            PartitionKey = [string]$PartitionKey
+            RowKey       = [string]$Item.TenantFilter
+            QueueId      = [string]$Item.QueueId
+            QueueType    = [string]$Item.QueueType
+            Data         = [string]$Json
         }
         Add-CIPPAzDataTableEntity @Table -Entity $GraphResults -Force | Out-Null
     } catch {
-        Write-Information "Queue Error: $($_.Exception.Message)"
+        Write-Warning "Queue Error: $($_.Exception.Message)"
+        #Write-Information ($GraphResults | ConvertTo-Json -Depth 10 -Compress)
         throw $_
     }
 }
diff --git a/Modules/CIPPCore/Public/GraphRequests/Get-GraphRequestList.ps1 b/Modules/CIPPCore/Public/GraphRequests/Get-GraphRequestList.ps1
@@ -138,9 +138,9 @@ function Get-GraphRequestList {
                 $Table = Get-CIPPTable -TableName $TableName
                 $Timestamp = (Get-Date).AddHours(-1).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffK')
                 if ($TenantFilter -eq 'AllTenants') {
-                    $Filter = "PartitionKey eq '{0}' and QueueType eq 'AllTenants' and Timestamp ge datetime'{1}'" -f $PartitionKey, $Timestamp
+                    $Filter = "PartitionKey eq '{0}' and Timestamp ge datetime'{1}'" -f $PartitionKey, $Timestamp
                 } else {
-                    $Filter = "PartitionKey eq '{0}' and Tenant eq '{1}' and Timestamp ge datetime'{2}'" -f $PartitionKey, $TenantFilter, $Timestamp
+                    $Filter = "PartitionKey eq '{0}' and (RowKey eq '{1}' or OriginalEntityId eq '{1}') and Timestamp ge datetime'{2}'" -f $PartitionKey, $TenantFilter, $Timestamp
                 }
                 $Rows = Get-CIPPAzDataTableEntity @Table -Filter $Filter
                 $Type = 'Cache'
@@ -337,8 +337,16 @@ function Get-GraphRequestList {
             }
         }
     } else {
-        $Rows | ForEach-Object {
-            $_.Data | ConvertFrom-Json
+        foreach ($Row in $Rows) {
+            if ($Row.Data) {
+                try {
+                    $Row.Data | ConvertFrom-Json -ErrorAction Stop
+                } catch {
+                    Write-Warning "Could not convert data to JSON: $($_.Exception.Message)"
+                    #Write-Information ($Row | ConvertTo-Json)
+                    continue
+                }
+            }
         }
     }
 }