Skip to content

Commit 8405755

Browse files
JohnDupreyJohnDuprey
committed
refactor: tests to fan-out > fan-in > fan-out like standards
1 parent 691fb6c commit 8405755

4 files changed

Lines changed: 115 additions & 59 deletions

File tree

‎Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Push-CIPPDBCacheApplyBatch.ps1‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
function Push-CIPPDBCacheApplyBatch {
2+
<#
3+
.SYNOPSIS
4+
Aggregate cache tasks from all tenants and start a flat execution orchestrator (Phase 2)
5+
6+
.DESCRIPTION
7+
PostExecution function for the DBCache pipeline. Receives aggregated results from the
8+
per-tenant CIPPDBCacheData list activities, flattens them into a single batch, and starts
9+
one orchestrator to execute all cache collection tasks across all tenants in parallel.
10+
11+
.FUNCTIONALITY
12+
Entrypoint
13+
#>
14+
param($Item)
15+
16+
try {
17+
# Aggregate all cache tasks from all tenant list activities
18+
$AllTasks = [System.Collections.Generic.List[object]]::new()
19+
20+
foreach ($TenantResult in $Item.Results) {
21+
foreach ($Batch in $TenantResult) {
22+
foreach ($Task in $Batch) {
23+
if ($Task -and $Task.FunctionName) {
24+
$AllTasks.Add($Task)
25+
}
26+
}
27+
}
28+
}
29+
30+
if ($AllTasks.Count -eq 0) {
31+
Write-Information 'No cache tasks to execute across all tenants'
32+
return @{ Success = $true; TaskCount = 0 }
33+
}
34+
35+
Write-Information "Aggregated $($AllTasks.Count) cache tasks from all tenants"
36+
37+
# Start a single flat orchestrator to execute all cache tasks
38+
$InputObject = [PSCustomObject]@{
39+
OrchestratorName = 'CIPPDBCacheExecute'
40+
Batch = @($AllTasks)
41+
SkipLog = $true
42+
}
43+
44+
# Add test run post-execution if flagged
45+
if ($Item.Parameters -and $Item.Parameters.TestRun -eq $true -and $Item.Parameters.TenantFilter) {
46+
$InputObject | Add-Member -NotePropertyName PostExecution -NotePropertyValue @{
47+
FunctionName = 'CIPPDBTestsRun'
48+
Parameters = @{
49+
TenantFilter = $Item.Parameters.TenantFilter
50+
}
51+
}
52+
}
53+
54+
$InstanceId = Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($InputObject | ConvertTo-Json -Depth 10 -Compress)
55+
Write-Information "Started flat cache execution orchestrator with ID = '$InstanceId' for $($AllTasks.Count) tasks"
56+
57+
return @{
58+
Success = $true
59+
TaskCount = $AllTasks.Count
60+
InstanceId = $InstanceId
61+
}
62+
63+
} catch {
64+
Write-Warning "Error in DBCache apply batch aggregation: $($_.Exception.Message)"
65+
return @{ Success = $false; Error = $_.Exception.Message }
66+
}
67+
}

‎Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Push-CIPPDBCacheData.ps1‎

Lines changed: 22 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,33 @@
11
function Push-CIPPDBCacheData {
22
<#
33
.SYNOPSIS
4-
Orchestrator function to collect and cache all data for a single tenant
4+
List cache collection tasks for a single tenant (Phase 1 of fan-out/fan-in)
55
66
.DESCRIPTION
7-
Builds a dynamic batch of cache collection tasks based on tenant license capabilities
7+
Checks tenant license capabilities and returns a list of cache collection work items.
8+
Does NOT start sub-orchestrators. The returned items are aggregated by the PostExecution
9+
function (CIPPDBCacheApplyBatch) and executed in a single flat orchestrator.
810
911
.FUNCTIONALITY
1012
Entrypoint
1113
#>
1214
[CmdletBinding()]
1315
param($Item)
14-
Write-Host "Starting cache collection orchestration for tenant: $($Item.TenantFilter) - Queue: $($Item.QueueName) (ID: $($Item.QueueId))"
16+
Write-Host "Building cache task list for tenant: $($Item.TenantFilter)"
1517
$TenantFilter = $Item.TenantFilter
1618
$QueueId = $Item.QueueId
1719

1820
try {
19-
Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message 'Starting database cache orchestration for tenant' -sev Info
20-
2121
# Check tenant capabilities for license-specific features
2222
$IntuneCapable = Test-CIPPStandardLicense -StandardName 'IntuneLicenseCheck' -TenantFilter $TenantFilter -RequiredCapabilities @('INTUNE_A', 'MDM_Services', 'EMS', 'SCCM', 'MICROSOFTINTUNEPLAN1') -SkipLog
2323
$ConditionalAccessCapable = Test-CIPPStandardLicense -StandardName 'ConditionalAccessLicenseCheck' -TenantFilter $TenantFilter -RequiredCapabilities @('AAD_PREMIUM', 'AAD_PREMIUM_P2') -SkipLog
2424
$AzureADPremiumP2Capable = Test-CIPPStandardLicense -StandardName 'AzureADPremiumP2LicenseCheck' -TenantFilter $TenantFilter -RequiredCapabilities @('AAD_PREMIUM_P2') -SkipLog
2525
$ExchangeCapable = Test-CIPPStandardLicense -StandardName 'ExchangeLicenseCheck' -TenantFilter $TenantFilter -RequiredCapabilities @('EXCHANGE_S_STANDARD', 'EXCHANGE_S_ENTERPRISE', 'EXCHANGE_S_STANDARD_GOV', 'EXCHANGE_S_ENTERPRISE_GOV', 'EXCHANGE_LITE') -SkipLog
2626

27-
Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message "License capabilities - Intune: $IntuneCapable, Conditional Access: $ConditionalAccessCapable, Azure AD Premium P2: $AzureADPremiumP2Capable, Exchange: $ExchangeCapable" -sev Info
27+
Write-Information "License capabilities for $TenantFilter - Intune: $IntuneCapable, CA: $ConditionalAccessCapable, P2: $AzureADPremiumP2Capable, Exchange: $ExchangeCapable"
2828

29-
# Build dynamic batch of cache collection tasks based on license capabilities
30-
$Batch = [System.Collections.Generic.List[object]]::new()
29+
# Build list of cache collection tasks based on license capabilities
30+
$Tasks = [System.Collections.Generic.List[object]]::new()
3131

3232
#region All Licenses - Basic tenant data collection
3333
$BasicCacheFunctions = @(
@@ -60,7 +60,7 @@ function Push-CIPPDBCacheData {
6060
)
6161

6262
foreach ($CacheFunction in $BasicCacheFunctions) {
63-
$Batch.Add(@{
63+
$Tasks.Add(@{
6464
FunctionName = 'ExecCIPPDBCache'
6565
Name = $CacheFunction
6666
TenantFilter = $TenantFilter
@@ -100,15 +100,15 @@ function Push-CIPPDBCacheData {
100100
)
101101

102102
foreach ($CacheFunction in $ExchangeCacheFunctions) {
103-
$Batch.Add(@{
103+
$Tasks.Add(@{
104104
FunctionName = 'ExecCIPPDBCache'
105105
Name = $CacheFunction
106106
TenantFilter = $TenantFilter
107107
QueueId = $QueueId
108108
})
109109
}
110110
} else {
111-
Write-Host 'Skipping Exchange Online data collection - tenant does not have required license'
111+
Write-Host "Skipping Exchange Online data collection for $TenantFilter - no required license"
112112
}
113113
#endregion Exchange Licensed
114114

@@ -121,15 +121,15 @@ function Push-CIPPDBCacheData {
121121
'UserRegistrationDetails'
122122
)
123123
foreach ($CacheFunction in $ConditionalAccessCacheFunctions) {
124-
$Batch.Add(@{
124+
$Tasks.Add(@{
125125
FunctionName = 'ExecCIPPDBCache'
126126
Name = $CacheFunction
127127
TenantFilter = $TenantFilter
128128
QueueId = $QueueId
129129
})
130130
}
131131
} else {
132-
Write-Host 'Skipping Conditional Access data collection - tenant does not have required license'
132+
Write-Host "Skipping Conditional Access data collection for $TenantFilter - no required license"
133133
}
134134
#endregion Conditional Access Licensed
135135

@@ -145,15 +145,15 @@ function Push-CIPPDBCacheData {
145145
'RoleManagementPolicies'
146146
)
147147
foreach ($CacheFunction in $P2CacheFunctions) {
148-
$Batch.Add(@{
148+
$Tasks.Add(@{
149149
FunctionName = 'ExecCIPPDBCache'
150150
Name = $CacheFunction
151151
TenantFilter = $TenantFilter
152152
QueueId = $QueueId
153153
})
154154
}
155155
} else {
156-
Write-Host 'Skipping Azure AD Premium P2 Identity Protection data collection - tenant does not have required license'
156+
Write-Host "Skipping Azure AD Premium P2 data collection for $TenantFilter - no required license"
157157
}
158158
#endregion Azure AD Premium P2
159159

@@ -167,49 +167,26 @@ function Push-CIPPDBCacheData {
167167
'DetectedApps'
168168
)
169169
foreach ($CacheFunction in $IntuneCacheFunctions) {
170-
$Batch.Add(@{
170+
$Tasks.Add(@{
171171
FunctionName = 'ExecCIPPDBCache'
172172
Name = $CacheFunction
173173
TenantFilter = $TenantFilter
174174
QueueId = $QueueId
175175
})
176176
}
177177
} else {
178-
Write-Host 'Skipping Intune data collection - tenant does not have required license'
178+
Write-Host "Skipping Intune data collection for $TenantFilter - no required license"
179179
}
180180
#endregion Intune Licensed
181181

182-
Write-Information "Built batch of $($Batch.Count) cache collection activities for tenant $TenantFilter"
183-
184-
# Start orchestration for this tenant's cache collection
185-
$InputObject = [PSCustomObject]@{
186-
OrchestratorName = "CIPPDBCacheTenant_$TenantFilter"
187-
Batch = @($Batch)
188-
SkipLog = $true
189-
}
182+
Write-Information "Built $($Tasks.Count) cache tasks for tenant $TenantFilter"
190183

191-
if ($Item.TestRun -eq $true) {
192-
$InputObject | Add-Member -NotePropertyName PostExecution -NotePropertyValue @{
193-
FunctionName = 'CIPPDBTestsRun'
194-
Parameters = @{
195-
TenantFilter = $TenantFilter
196-
}
197-
}
198-
}
199-
200-
$InstanceId = Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($InputObject | ConvertTo-Json -Depth 5 -Compress)
201-
Write-Information "Started cache collection orchestration for $TenantFilter with ID = '$InstanceId'"
202-
Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message "Started cache collection orchestration with $($Batch.Count) activities. Instance ID: $InstanceId" -sev Info
203-
204-
return @{
205-
InstanceId = $InstanceId
206-
BatchCount = $Batch.Count
207-
Message = "Cache collection orchestration started for $TenantFilter"
208-
}
184+
# Return the task list — the PostExecution function will aggregate and start a flat orchestrator
185+
return @($Tasks)
209186

210187
} catch {
211188
$ErrorMessage = Get-CippException -Exception $_
212-
Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message "Failed to start cache collection orchestration: $($ErrorMessage.NormalizedError)" -sev Error -LogData $ErrorMessage
213-
throw $ErrorMessage
189+
Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message "Failed to build cache task list: $($ErrorMessage.NormalizedError)" -sev Error -LogData $ErrorMessage
190+
return @()
214191
}
215192
}

‎Modules/CIPPCore/Public/Entrypoints/HTTP Functions/Invoke-ExecTestRun.ps1‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,19 @@ function Invoke-ExecTestRun {
1919
TenantFilter = $TenantFilter
2020
QueueId = $Queue.RowKey
2121
QueueName = "Cache - $TenantFilter"
22-
TestRun = $true
2322
}
2423
)
2524
$InputObject = [PSCustomObject]@{
2625
OrchestratorName = 'TestDataCollectionAndRun'
2726
Batch = $Batch
2827
SkipLog = $false
28+
PostExecution = @{
29+
FunctionName = 'CIPPDBCacheApplyBatch'
30+
Parameters = @{
31+
TestRun = $true
32+
TenantFilter = $TenantFilter
33+
}
34+
}
2935
}
3036

3137
$InstanceId = Start-CIPPOrchestrator -InputObject $InputObject

‎Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-CIPPDBCacheOrchestrator.ps1‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@ function Start-CIPPDBCacheOrchestrator {
44
Orchestrates database cache collection across all tenants
55
66
.DESCRIPTION
7-
Creates per-tenant jobs to collect and cache Microsoft Graph data
7+
Uses a two-phase fan-out/fan-in pattern (matching Standards):
8+
Phase 1: Fan out CIPPDBCacheData activities per tenant to check licenses and build task lists
9+
Phase 2: PostExecution aggregates all tasks and starts a single flat orchestrator to execute them
810
911
.FUNCTIONALITY
1012
Entrypoint
@@ -22,24 +24,28 @@ function Start-CIPPDBCacheOrchestrator {
2224
return
2325
}
2426

25-
$TaskCount = $TenantList.Count
26-
27-
$Queue = New-CippQueueEntry -Name 'Database Cache Collection' -TotalTasks $TaskCount
28-
$Batch = [system.collections.generic.list[object]]::new()
29-
foreach ($Tenant in $TenantList) {
30-
$Batch.Add([PSCustomObject]@{
31-
FunctionName = 'CIPPDBCacheData'
32-
TenantFilter = $Tenant.defaultDomainName
33-
QueueId = $Queue.RowKey
34-
QueueName = "DB Cache - $($Tenant.defaultDomainName)"
35-
})
27+
$Queue = New-CippQueueEntry -Name 'Database Cache Collection' -TotalTasks $TenantList.Count
28+
29+
# Phase 1: Build per-tenant list activities (license check + task list generation)
30+
$Batch = foreach ($Tenant in $TenantList) {
31+
[PSCustomObject]@{
32+
FunctionName = 'CIPPDBCacheData'
33+
TenantFilter = $Tenant.defaultDomainName
34+
QueueId = $Queue.RowKey
35+
QueueName = "DB Cache - $($Tenant.defaultDomainName)"
36+
}
3637
}
38+
3739
Write-Host "Created queue $($Queue.RowKey) for database cache collection of $($TenantList.Count) tenants"
38-
Write-Host "Starting batch of $($Batch.Count) cache collection activities"
40+
41+
# Phase 2 via PostExecution: Aggregate all task lists and start flat execution orchestrator
3942
$InputObject = [PSCustomObject]@{
4043
Batch = @($Batch)
4144
OrchestratorName = 'CIPPDBCacheOrchestrator'
4245
SkipLog = $false
46+
PostExecution = @{
47+
FunctionName = 'CIPPDBCacheApplyBatch'
48+
}
4349
}
4450

4551
Start-CIPPOrchestrator -InputObject $InputObject

0 commit comments

Comments
 (0)