Skip to content

WIP: Add concurrency for parent packages #2016

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
alerickson wants to merge 16 commits into
base: master
Choose a base branch
from
+227 −192
Open

WIP: Add concurrency for parent packages #2016

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
ba2d230
Implement unimplemented concurrency methods for all server classes
alerickson Jun 30, 2026
fbb940f
build fixes
alerickson Jun 30, 2026
a4730af
Fix thread-safety in ContainerRegistryServerAPICalls.InstallPackageAsync
Copilot Jul 1, 2026
f800970
Fix copilot commit for container registry
alerickson Jul 1, 2026
6ab7367
Avoid cmdlet stream writes in NuGet FindVersionAsync path
Copilot Jul 1, 2026
dbd1689
Fix V3 async helper logging to use debug queues
Copilot Jul 1, 2026
efb000d
Potential fix for pull request finding
alerickson Jul 1, 2026
0e9c684
Fix specific-version async dependency error handling path
Copilot Jul 1, 2026
e3c1af5
Scope async dependency queue handling to current operation
Copilot Jul 1, 2026
9b92e77
Fix version-range async path: reset errRecord and flush concurrent qu…
Copilot Jul 1, 2026
900db22
build fixes
alerickson Jul 1, 2026
4cdbea8
Incorporate code review changes - complete TODOs and remove unneeded …
alerickson Jul 21, 2026
40138f1
Add concurrency for parent packages
alerickson Jul 24, 2026
8c5e184
Merge branch 'master' of https://github.com/powershell/PSResourceGet …
alerickson Jul 31, 2026
06e8a86
Use ConcurrentQueue for commented out debug msg
alerickson Jul 31, 2026
90d11f7
Bug fix for filtering on version
alerickson Jul 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 43 additions & 18 deletions src/code/FindHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,6 @@ internal class FindHelper
// If running 'Install-PSResource Az, TestModule, NewTestModule', it will contain one parent and its dependencies.
private ConcurrentDictionary<string, List<string>> _packagesFound;

// Creates a new instance of depPkgsFound each time FindDependencyPackages() is called.
// This will eventually return the PSResourceInfo object to the main cmdlet class.
private ConcurrentDictionary<string, PSResourceInfo> depPkgsFound;

// Contains the latest found version of a particular package.
private ConcurrentDictionary<string, PSResourceInfo> _knownLatestPkgVersion;

Expand Down Expand Up @@ -1060,13 +1056,36 @@ private IEnumerable<PSResourceInfo> SearchByNames(ServerApiCall currentServer, R
// After retrieving all packages find their dependencies
if (_includeDependencies)
{
foreach (PSResourceInfo currentPkg in parentPkgs)
// Resolving each parent package's dependency closure is independent work, so do it concurrently.
// yield return cannot be used inside Parallel.ForEach, so collect results into a thread-safe bag first.
ConcurrentBag<PSResourceInfo> dependencyPkgs = new ConcurrentBag<PSResourceInfo>();
int processorCount = Environment.ProcessorCount;
int maxDegreeOfParallelism = processorCount * 4;
if (parentPkgs.Count > processorCount)
{
Parallel.ForEach(parentPkgs, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, currentPkg =>
{
foreach (PSResourceInfo pkgDep in FindDependencyPackages(currentServer, currentResponseUtil, currentPkg, repository))
{
dependencyPkgs.Add(pkgDep);
}
});
}
else
{
foreach (PSResourceInfo pkgDep in FindDependencyPackages(currentServer, currentResponseUtil, currentPkg, repository))
foreach (PSResourceInfo currentPkg in parentPkgs)
{
yield return pkgDep;
foreach (PSResourceInfo pkgDep in FindDependencyPackages(currentServer, currentResponseUtil, currentPkg, repository))
{
dependencyPkgs.Add(pkgDep);
}
}
}

foreach (PSResourceInfo pkgDep in dependencyPkgs)
{
yield return pkgDep;
}
}
}

Expand Down Expand Up @@ -1158,15 +1177,17 @@ private string FormatPkgVersionString(PSResourceInfo pkg)

internal IEnumerable<PSResourceInfo> FindDependencyPackages(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository)
{
depPkgsFound = new ConcurrentDictionary<string, PSResourceInfo>();
// Use a local instance so multiple parent packages can resolve their dependency closures concurrently
// without racing on shared state.
ConcurrentDictionary<string, PSResourceInfo> depPkgsFound = new ConcurrentDictionary<string, PSResourceInfo>();
_cmdletPassedIn.WriteDebug($"In FindHelper::FindDependencyPackages() - {currentPkg.Name}");
FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository);
FindDependencyPackagesHelper(currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound);
Comment on lines +1180 to +1184

return depPkgsFound.Values.ToList();
}

// Method 2
internal void FindDependencyPackagesHelper(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository)
internal void FindDependencyPackagesHelper(ServerApiCall currentServer, ResponseUtil currentResponseUtil, PSResourceInfo currentPkg, PSRepositoryInfo repository, ConcurrentDictionary<string, PSResourceInfo> depPkgsFound)
{
ConcurrentQueue<ErrorRecord> errorMsgs = new ConcurrentQueue<ErrorRecord>();
ConcurrentQueue<string> verboseMsgs = new ConcurrentQueue<string>();
Expand All @@ -1185,15 +1206,15 @@ internal void FindDependencyPackagesHelper(ServerApiCall currentServer, Response
Parallel.ForEach(currentPkg.Dependencies, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, dep =>
{
debugMsgs.Enqueue($"Finding dependency '{dep.Name}' version range '{dep.VersionRange}'");
FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs);
FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs);
});
}
else
{
foreach (var dep in currentPkg.Dependencies)
{
debugMsgs.Enqueue($"Finding dependency '{dep.Name}' version range '{dep.VersionRange}'");
FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs);
FindDependencyPackageVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs);
}
}

Expand All @@ -1208,6 +1229,7 @@ private void FindDependencyPackageVersion(
ResponseUtil currentResponseUtil,
PSResourceInfo currentPkg,
PSRepositoryInfo repository,
ConcurrentDictionary<string, PSResourceInfo> depPkgsFound,
ConcurrentQueue<ErrorRecord> errorMsgs,
ConcurrentQueue<string> warningMsgs,
ConcurrentQueue<string> debugMsgs,
Expand All @@ -1228,7 +1250,7 @@ private void FindDependencyPackageVersion(
else
{
// Find this version from the server
depPkg = FindDependencyWithLowerBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs);
depPkg = FindDependencyWithLowerBound(dep, currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs);
}
}
else if (dep.VersionRange.HasLowerBound && dep.VersionRange.MinVersion.Equals(dep.VersionRange.MaxVersion))
Expand All @@ -1245,7 +1267,7 @@ private void FindDependencyPackageVersion(
}
else
{
depPkg = FindDependencyWithSpecificVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs);
depPkg = FindDependencyWithSpecificVersion(dep, currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs);
}
}
else
Expand All @@ -1261,7 +1283,7 @@ private void FindDependencyPackageVersion(
}
else
{
depPkg = FindDependencyWithUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, errorMsgs, warningMsgs, debugMsgs, verboseMsgs);
depPkg = FindDependencyWithUpperBound(dep, currentServer, currentResponseUtil, currentPkg, repository, depPkgsFound, errorMsgs, warningMsgs, debugMsgs, verboseMsgs);
}
}
}
Expand All @@ -1273,6 +1295,7 @@ private PSResourceInfo FindDependencyWithSpecificVersion(
ResponseUtil currentResponseUtil,
PSResourceInfo currentPkg,
PSRepositoryInfo repository,
ConcurrentDictionary<string, PSResourceInfo> depPkgsFound,
ConcurrentQueue<ErrorRecord> errorMsgs,
ConcurrentQueue<string> warningMsgs,
ConcurrentQueue<string> debugMsgs,
Expand Down Expand Up @@ -1351,7 +1374,7 @@ private PSResourceInfo FindDependencyWithSpecificVersion(
// This will eventually return the PSResourceInfo object to the main cmdlet class.
debugMsgs.Enqueue($"Adding'{key}' to list of dependency packages found");
depPkgsFound.TryAdd(key, depPkg);
FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository);
FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository, depPkgsFound);
}
}
}
Expand All @@ -1366,6 +1389,7 @@ private PSResourceInfo FindDependencyWithLowerBound(
ResponseUtil currentResponseUtil,
PSResourceInfo currentPkg,
PSRepositoryInfo repository,
ConcurrentDictionary<string, PSResourceInfo> depPkgsFound,
ConcurrentQueue<ErrorRecord> errorMsgs,
ConcurrentQueue<string> warningMsgs,
ConcurrentQueue<string> debugMsgs,
Expand Down Expand Up @@ -1419,7 +1443,7 @@ private PSResourceInfo FindDependencyWithLowerBound(
// This will eventually return the PSResourceInfo object to the main cmdlet class.
debugMsgs.Enqueue($"Adding'{key}' to list of dependency packages found");
depPkgsFound.TryAdd(key, depPkg);
FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository);
FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository, depPkgsFound);
}
}
}
Expand All @@ -1434,6 +1458,7 @@ private PSResourceInfo FindDependencyWithUpperBound(
ResponseUtil currentResponseUtil,
PSResourceInfo currentPkg,
PSRepositoryInfo repository,
ConcurrentDictionary<string, PSResourceInfo> depPkgsFound,
ConcurrentQueue<ErrorRecord> errorMsgs,
ConcurrentQueue<string> warningMsgs,
ConcurrentQueue<string> debugMsgs,
Expand Down Expand Up @@ -1490,7 +1515,7 @@ private PSResourceInfo FindDependencyWithUpperBound(
// This will eventually return the PSResourceInfo object to the main cmdlet class.
debugMsgs.Enqueue($"Adding'{key}' to list of dependency packages found");
depPkgsFound.TryAdd(key, depPkg);
FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository);
FindDependencyPackagesHelper(currentServer, currentResponseUtil, depPkg, repository, depPkgsFound);
}
}
}
Expand Down
Loading
Loading