[Azure Search] Boost freshness (#521)

Add a scoring profile that gives reasonable relevancy by favoring packages with high download counts that have been updated recently. I tested this by indexing PROD and then trying queries from our OneNote by hand.

Addresses #6904

src/NuGet.Services.AzureSearch/IndexBuilder.cs

@@ -128,8 +128,8 @@ private Index InitializeIndex<TDocument>(string name, bool addScoringProfile)
 
             if (addScoringProfile)
             {
-                index.ScoringProfiles = new List<ScoringProfile> { DownloadCountBoosterProfile.Instance };
-                index.DefaultScoringProfile = DownloadCountBoosterProfile.Name;
+                index.ScoringProfiles = new List<ScoringProfile> { DefaultScoringProfile.Instance };
+                index.DefaultScoringProfile = DefaultScoringProfile.Name;
             }
 
             return index;

src/NuGet.Services.AzureSearch/Models/BaseMetadataDocument.cs

@@ -56,6 +56,7 @@ public abstract class BaseMetadataDocument : CommittedDocument, IBaseMetadataDoc
         public string ProjectUrl { get; set; }
 
         [IsSortable]
+        [IsFilterable]
         public DateTimeOffset? Published { get; set; }
 
         public string ReleaseNotes { get; set; }

src/NuGet.Services.AzureSearch/Models/SearchDocument.cs

@@ -21,6 +21,9 @@ public class Full : UpdateLatest
         {
             [IsFilterable]
             public long? TotalDownloadCount { get; set; }
+
+            [IsFilterable]
+            public double? LogBase2DownloadCount { get; set; }
         }
 
         /// <summary>

src/NuGet.Services.AzureSearch/NuGet.Services.AzureSearch.csproj

@@ -57,7 +57,7 @@
     <Compile Include="PackageIdToOwnersBuilder.cs" />
     <Compile Include="IOwnerDataClient.cs" />
     <Compile Include="OwnerDataClient.cs" />
-    <Compile Include="ScoringProfiles\DownloadCountBoosterProfile.cs" />
+    <Compile Include="ScoringProfiles\DefaultScoringProfile.cs" />
     <Compile Include="Owners2AzureSearch\Owners2AzureSearchCommand.cs" />
     <Compile Include="SearchService\AzureSearchQueryBuilder.cs" />
     <Compile Include="BlobContainerBuilder.cs" />

src/NuGet.Services.AzureSearch/ScoringProfiles/DefaultScoringProfile.cs

@@ -0,0 +1,55 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.Azure.Search.Models;
+using NuGet.Services.AzureSearch.SearchService;
+
+namespace NuGet.Services.AzureSearch.ScoringProfiles
+{
+    public static class DefaultScoringProfile
+    {
+        public const string Name = "nuget_scoring_profile";
+
+        public static readonly ScoringProfile Instance = new ScoringProfile(
+            Name,
+            textWeights: new TextWeights
+            {
+                Weights = new Dictionary<string, double>
+                {
+                    // Exact match of the package id should be boosted the highest,
+                    // followed by a tokenized match on the package id.
+                    { IndexFields.PackageId, 10 },
+                    { IndexFields.TokenizedPackageId, 5 },
+                }
+            },
+            functions: new List<ScoringFunction>()
+            {
+                // Greatly boost results with high download counts. We score off the log of the download count
+                // with linear interpolation so that the boost slows down at higher download counts. We cannot
+                // use the raw download count with a log interpolation as that would result in a large boosting
+                // range, which would need to be offset by an unmanageably high boosting factor.
+                new MagnitudeScoringFunction(
+                    fieldName: IndexFields.Search.LogBase2DownloadCount,
+                    boost: 100.0,
+                    parameters: new MagnitudeScoringParameters(
+                        boostingRangeStart: 0,
+                        boostingRangeEnd: Math.Log(999_999_999_999, 2),
+                        shouldBoostBeyondRangeByConstant: true),
+                    interpolation: ScoringFunctionInterpolation.Linear),
+
+                // Boost results with a recent published date. We use a quadatric interpolation
+                // so that the boost decreases faster as the publish date nears the end of the boost range.
+                new FreshnessScoringFunction(
+                    fieldName: IndexFields.Published,
+                    boost: 5.0,
+                    boostingDuration: TimeSpan.FromDays(365),
+                    interpolation: ScoringFunctionInterpolation.Quadratic),
+            },
+
+            // The scores of each Scoring Function should be summed together before multiplying the base relevance scores.
+            // See: https://stackoverflow.com/questions/41427940/how-do-scoring-profiles-generate-scores-in-azure-search
+            functionAggregation: ScoringFunctionAggregation.Sum);
+    }
+}

src/NuGet.Services.AzureSearch/SearchDocumentBuilder.cs

@@ -201,6 +201,7 @@ public SearchDocument.Full FullFromDb(
                 owners: owners);
             DocumentUtilities.PopulateMetadata(document, packageId, package);
             document.TotalDownloadCount = totalDownloadCount;
+            document.LogBase2DownloadCount = Math.Log(totalDownloadCount, 2.0);
 
             return document;
         }

src/NuGet.Services.AzureSearch/SearchService/IndexFields.cs

@@ -15,6 +15,7 @@ private static string Name(string input)
         }
 
         public static readonly string Authors = Name(nameof(BaseMetadataDocument.Authors));
+        public static readonly string Created = Name(nameof(BaseMetadataDocument.Created));
         public static readonly string Description = Name(nameof(BaseMetadataDocument.Description));
         public static readonly string LastEdited = Name(nameof(BaseMetadataDocument.LastEdited));
         public static readonly string NormalizedVersion = Name(nameof(BaseMetadataDocument.NormalizedVersion));
@@ -32,6 +33,7 @@ public static class Search
             public static readonly string Owners = Name(nameof(SearchDocument.Full.Owners));
             public static readonly string SearchFilters = Name(nameof(SearchDocument.UpdateLatest.SearchFilters));
             public static readonly string TotalDownloadCount = Name(nameof(SearchDocument.Full.TotalDownloadCount));
+            public static readonly string LogBase2DownloadCount = Name(nameof(SearchDocument.Full.LogBase2DownloadCount));
             public static readonly string Versions = Name(nameof(SearchDocument.UpdateLatest.Versions));
         }
     }

tests/NuGet.Services.AzureSearch.Tests/SearchDocumentBuilderFacts.cs

@@ -493,6 +493,7 @@ public async Task SetsExpectedProperties(SearchFilters searchFilters, string exp
     {
       ""@search.action"": ""upload"",
       ""totalDownloadCount"": 1001,
+      ""logBase2DownloadCount"": 9.9672262588359928,
       ""owners"": [
         ""Microsoft"",
         ""azure-sdk""