添加Powershell作业以从分页的rest API中并行检索

Question

我花了大约30个小时来尝试添加各种方法来实现对我的API调用的并行处理。最明显的是将代码放入工作中，但是我没有运气。有什么想法吗？

Start-Transcript -Path "$PSScriptRoot\Errorlog.txt"

$WondeObjectsArray = Import-CSV $PSScriptRoot\WondeID.csv
$EndpointObjectArray = Import-CSV $PSScriptRoot\Endpoints.csv

$headers = New-Object "System.Collections.Generic.Dictionary[[String],[String]]"
$headers.Add("REDACTED", "REDACTED")

# School loop

Foreach ($object in $WondeObjectsArray) {
    $SchoolName = $object.School
    $TrustName = $object.Trust
    $WondeID = $object."Wonde ID"

# Create data structure
If(!(test-path "$PSScriptRoot\$TrustName")) 
    {New-Item -Path "$PSScriptRoot\$TrustName" -ItemType Directory}
If(!(test-path "$PSScriptRoot\$TrustName\$SchoolName")) 
    {New-Item -Path "$PSScriptRoot\$TrustName\$SchoolName" -ItemType Directory}


# Endpoint request loop 
foreach($Eobject in $EndpointObjectArray){
    $JsonName = $Eobject.JsonName
    $Table = $Eobject.Table
    $Method = $Eobject.Url_Method

# First response
$response = Invoke-RestMethod "https://api.wonde.com/v1.0/schools/$WondeID/$Table$Method&$Page" -Method 'GET' -Headers $headers -Body $body
    $concat = $response.data
#Pagination loop
While ($response.meta.pagination.next){
    $response = Invoke-RestMethod $response.meta.Pagination.next -Method 'GET' -Headers $headers -Body $Body
    $concat = $concat + $response.data
} #pagination loop end

# Concatenate completed request
    $concat | ConvertTo-Json | Out-File "$PSScriptRoot\$TrustName\$SchoolName\$JsonName.json"

} # Endpoint request loop end
} # School loop end

Stop-Transcript

Answer 1

没关系，我想通了。我将要并行运行的代码转换为脚本块，使用param（）将其传递给了paramiters。在开始的工作行中，我将变量包含在-ArgumentList中。

我的问题实际上是，当我传递$ PSScriptRoot而不是默认文件位置时，当您开始一项新工作时，它将脚本文件路径更改为C：/。

Start-Transcript -Path "$PSScriptRoot\Errorlog.txt"

$PathRoot = $PSScriptRoot
$WondeObjectsArray = Import-CSV $PSScriptRoot\WondeID.csv
$EndpointObjectArray = Import-CSV $PSScriptRoot\Endpoints.csv

$headers = New-Object "System.Collections.Generic.Dictionary[[String],[String]]"
$headers.Add("REDACTED", "REDACTED")

# School loop
Foreach ($object in $WondeObjectsArray) {
    $SchoolName = $object.School
    $TrustName = $object.Trust
    $WondeID = $object."Wonde ID"

# Create data structure
If(!(test-path "$PSScriptRoot\$TrustName")) 
    {New-Item -Path "$PSScriptRoot\$TrustName" -ItemType Directory}
If(!(test-path "$PSScriptRoot\$TrustName\$SchoolName")) 
    {New-Item -Path "$PSScriptRoot\$TrustName\$SchoolName" -ItemType Directory}

# Endpoint request loop 
foreach ($Eobject in $EndpointObjectArray){
    $JsonName = $Eobject.JsonName
    $Table = $Eobject.Table
    $Method = $Eobject.Url_Method

$scriptblock = {
param ($WondeID, $Table, $Method, $headers, $trustName, $SchoolName, $JsonName, $PathRoot)

write-host "$TrustName : $SchoolName : $JsonName is complete!"

# First response
$response = Invoke-RestMethod "https://api.wonde.com/v1.0/schools/$WondeID/$Table$Method&$Page" -Method 'GET' -Headers $headers -Body $body
    $concat = $response.data
#Pagination loop
While ($response.meta.pagination.next){
    $response = Invoke-RestMethod $response.meta.Pagination.next -Method 'GET' -Headers $headers -Body $Body
    $concat = $concat + $response.data
} #pagination loop end

# Concatenate completed request
    $concat | ConvertTo-Json | Out-File "$PathRoot\$TrustName\$SchoolName\$JsonName.json"

}

Start-Job $scriptblock -Name "$SchoolName&$JsonName" -ArgumentList $WondeID, $Table, $Method, $headers, $trustName, $SchoolName, $JsonName, $PathRoot

} # Endpoint request loop end

Get-Job | Wait-Job | Receive-Job

Get-Job | Remove-Job 

} # School loop end

Stop-Transcript