Skip to content

Commit

Permalink
Files regex processing boost (#3953)
Browse files Browse the repository at this point in the history 
* Changed recursive wildcards regex pattern to faster form
Added exclude pattern TPL provessing
Fixed linq lazy regex objects spawn

* Fixed unit tests. AsParallel not guaranty result order.

* Fixed windows regex for network paths

* Reworked CreatePackageWithNuspecIncludeExcludeWithWildcards test accordingly to review request
  • Loading branch information
@BlackGad
BlackGad authored Mar 29, 2021
1 parent 87194dd commit c6ca6eb
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 24 deletions.
10 changes: 5 additions & 5 deletions src/NuGet.Core/NuGet.Common/PathUtil/PathResolver.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ public static IEnumerable<T> GetMatches<T>(
Func<T, string> getPath,
IEnumerable<string> wildcards)
{
IEnumerable<Regex> filters = wildcards.Select(WildcardToRegex);
List<Regex> filters = wildcards.Select(WildcardToRegex).ToList();

return source.Where(item =>
return source.AsParallel().Where(item =>
{
string path = getPath(item);
return filters.Any(f => f.IsMatch(path));
Expand Down Expand Up @@ -79,7 +79,7 @@ private static Regex WildcardToRegex(string wildcard)
// regex wildcard adjustments for *nix-style file systems
pattern = pattern
.Replace(@"\.\*\*", @"\.[^/.]*") // .** should not match on ../file or ./file but will match .file
.Replace(@"\*\*/", "(.+/)*") //For recursive wildcards /**/, include the current directory.
.Replace(@"\*\*/", "/?([^/]+/)*?") //For recursive wildcards /**/, include the current directory.
.Replace(@"\*\*", ".*") // For recursive wildcards that don't end in a slash e.g. **.txt would be treated as a .txt file at any depth
.Replace(@"\*", @"[^/]*(/)?") // For non recursive searches, limit it any character that is not a directory separator
.Replace(@"\?", "."); // ? translates to a single any character
Expand All @@ -90,7 +90,7 @@ private static Regex WildcardToRegex(string wildcard)
pattern = pattern
.Replace("/", @"\\") // On Windows, / is treated the same as \.
.Replace(@"\.\*\*", @"\.[^\\.]*") // .** should not match on ../file or ./file but will match .file
.Replace(@"\*\*\\", @"(.+\\)*") //For recursive wildcards \**\, include the current directory.
.Replace(@"\*\*\\", @"(\\\\)?([^\\]+\\)*?") //For recursive wildcards \**\, include the current directory.
.Replace(@"\*\*", ".*") // For recursive wildcards that don't end in a slash e.g. **.txt would be treated as a .txt file at any depth
.Replace(@"\*", @"[^\\]*(\\)?") // For non recursive searches, limit it any character that is not a directory separator
.Replace(@"\?", "."); // ? translates to a single any character
Expand Down Expand Up @@ -146,7 +146,7 @@ public static IEnumerable<SearchPathResult> PerformWildcardSearch(

// Starting from the base path, enumerate over all files and match it using the wildcard expression provided by the user.
// Note: We use Directory.GetFiles() instead of Directory.EnumerateFiles() here to support Mono
IEnumerable<SearchPathResult> matchedFiles = from file in Directory.GetFiles(normalizedBasePath, "*.*", searchOption)
IEnumerable<SearchPathResult> matchedFiles = from file in Directory.GetFiles(normalizedBasePath, "*.*", searchOption).AsParallel()
where searchRegex.IsMatch(file)
select new SearchPathResult(file, isFile: true);

Expand Down
40 changes: 32 additions & 8 deletions test/NuGet.Core.Tests/NuGet.Common.Test/PathResolverTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,10 @@ public void GetMatches_WithDotGlobstar_ReturnsMatches()
IEnumerable<string> actualResults = PathResolver.GetMatches(sources, source => source, wildcards);
IEnumerable<string> expectedResults = GetPlatformSpecificPaths(new[] { ".c" });

Assert.Equal(expectedResults, actualResults);
IEnumerable<string> orderedActualResults = actualResults.OrderBy(path => path);
IEnumerable<string> orderedExpectedResults = expectedResults.OrderBy(path => path);

Assert.Equal(orderedExpectedResults, orderedActualResults);
}

[Fact]
Expand All @@ -146,7 +149,10 @@ public void GetMatches_WithGlobstarSlash_ReturnsMatches()
IEnumerable<string> actualResults = PathResolver.GetMatches(sources, source => source, wildcards);
IEnumerable<string> expectedResults = GetPlatformSpecificPaths(new[] { "a{0}d", "a{0}b{0}d", "a{0}b{0}c{0}d" });

Assert.Equal(expectedResults, actualResults);
IEnumerable<string> orderedActualResults = actualResults.OrderBy(path => path);
IEnumerable<string> orderedExpectedResults = expectedResults.OrderBy(path => path);

Assert.Equal(orderedExpectedResults, orderedActualResults);
}

[Fact]
Expand All @@ -157,7 +163,10 @@ public void GetMatches_WithGlobstarDotFileName_ReturnsFileNameMatches()
IEnumerable<string> actualResults = PathResolver.GetMatches(sources, source => source, wildcards);
IEnumerable<string> expectedResults = GetPlatformSpecificPaths(new[] { ".{0}.c", "a{0}.c", "a{0}{0}.c", "b.c", "a{0}b{0}bc.c", "a{0}b{0}.c" });

Assert.Equal(expectedResults, actualResults);
IEnumerable<string> orderedActualResults = actualResults.OrderBy(path => path);
IEnumerable<string> orderedExpectedResults = expectedResults.OrderBy(path => path);

Assert.Equal(orderedExpectedResults, orderedActualResults);
}

[Fact]
Expand All @@ -168,7 +177,10 @@ public void GetMatches_WithGlobstarSlashDotFileName_ReturnsFileNameMatches()
IEnumerable<string> actualResults = PathResolver.GetMatches(sources, source => source, wildcards);
IEnumerable<string> expectedResults = GetPlatformSpecificPaths(new[] { ".{0}.c", "a{0}.c", "a{0}{0}.c", "b.c", "a{0}b{0}bc.c", "a{0}b{0}.c" });

Assert.Equal(expectedResults, actualResults);
IEnumerable<string> orderedActualResults = actualResults.OrderBy(path => path);
IEnumerable<string> orderedExpectedResults = expectedResults.OrderBy(path => path);

Assert.Equal(orderedExpectedResults, orderedActualResults);
}

[Fact]
Expand All @@ -179,7 +191,10 @@ public void GetMatches_WithGlobstarSlashFileName_ReturnsFileNameMatches()
IEnumerable<string> actualResults = PathResolver.GetMatches(sources, source => source, wildcards);
IEnumerable<string> expectedResults = GetPlatformSpecificPaths(new[] { ".{0}c", "a{0}c" });

Assert.Equal(expectedResults, actualResults);
IEnumerable<string> orderedActualResults = actualResults.OrderBy(path => path);
IEnumerable<string> orderedExpectedResults = expectedResults.OrderBy(path => path);

Assert.Equal(orderedExpectedResults, orderedActualResults);
}

[Fact]
Expand All @@ -190,7 +205,10 @@ public void GetMatches_WithGlobstar_ReturnsMatches()
IEnumerable<string> actualResults = PathResolver.GetMatches(sources, source => source, wildcards);
IEnumerable<string> expectedResults = GetPlatformSpecificPaths(new[] { "a{0}d", "a{0}b{0}d", "a{0}b{0}c{0}d" });

Assert.Equal(expectedResults, actualResults);
IEnumerable<string> orderedActualResults = actualResults.OrderBy(path => path);
IEnumerable<string> orderedExpectedResults = expectedResults.OrderBy(path => path);

Assert.Equal(orderedExpectedResults, orderedActualResults);
}

[Fact]
Expand All @@ -201,7 +219,10 @@ public void GetMatches_WithStar_ReturnsMatches()
IEnumerable<string> actualResults = PathResolver.GetMatches(sources, source => source, wildcards);
IEnumerable<string> expectedResults = GetPlatformSpecificPaths(new[] { "a{0}b{0}c{0}d" });

Assert.Equal(expectedResults, actualResults);
IEnumerable<string> orderedActualResults = actualResults.OrderBy(path => path);
IEnumerable<string> orderedExpectedResults = expectedResults.OrderBy(path => path);

Assert.Equal(orderedExpectedResults, orderedActualResults);
}

[Fact]
Expand All @@ -212,7 +233,10 @@ public void GetMatches_WithQuestionMark_ReturnsMatches()
IEnumerable<string> actualResults = PathResolver.GetMatches(sources, source => source, wildcards);
string[] expectedResults = new[] { "abc", "adc" };

Assert.Equal(expectedResults, actualResults);
IEnumerable<string> orderedActualResults = actualResults.OrderBy(path => path);
IEnumerable<string> orderedExpectedResults = expectedResults.OrderBy(path => path);

Assert.Equal(orderedExpectedResults, orderedActualResults);
}

[Fact]
Expand Down
29 changes: 18 additions & 11 deletions test/NuGet.Core.Tests/NuGet.Packaging.Test/PackageBuilderTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -345,22 +345,29 @@ public void CreatePackageWithNuspecIncludeExcludeWithWildcards(string source, st
builder.AddFiles(directory.Path, source, destination, exclude);

// Assert
Assert.Collection(builder.Files,
file =>
var expectedResults = new[]
{
new
{
Assert.Equal(string.Format("Content{0}file1.txt", Path.DirectorySeparatorChar), file.Path);
Assert.Equal("file1.txt", file.EffectivePath);
Path = string.Format("Content{0}file1.txt", Path.DirectorySeparatorChar),
EffectivePath = "file1.txt"
},
file =>
new
{
Assert.Equal(string.Format("Content{0}dir1{0}file1.txt", Path.DirectorySeparatorChar), file.Path);
Assert.Equal(string.Format("dir1{0}file1.txt", Path.DirectorySeparatorChar), file.EffectivePath);
Path = string.Format("Content{0}dir1{0}file1.txt", Path.DirectorySeparatorChar),
EffectivePath = string.Format("dir1{0}file1.txt", Path.DirectorySeparatorChar)
},
file =>
new
{
Assert.Equal(string.Format("Content{0}dir1{0}dir2{0}file1.txt", Path.DirectorySeparatorChar), file.Path);
Assert.Equal(string.Format("dir1{0}dir2{0}file1.txt", Path.DirectorySeparatorChar), file.EffectivePath);
});
Path = string.Format("Content{0}dir1{0}dir2{0}file1.txt", Path.DirectorySeparatorChar),
EffectivePath = string.Format("dir1{0}dir2{0}file1.txt", Path.DirectorySeparatorChar)
}
};

var orderedExpectedResults = expectedResults.OrderBy(i => i.Path);
var orderedActualResults = builder.Files.Select(f => new { f.Path, f.EffectivePath }).OrderBy(i => i.Path);

Assert.Equal(orderedExpectedResults, orderedActualResults);
}
}

Expand Down

0 comments on commit c6ca6eb

Please sign in to comment.