Add option to control whether project files are indexed

Summary: This enables us to use a JK as a kill-switch for the new 'project file indexing' feature

Reviewed By: pepeiborra

Differential Revision: D53578449

fbshipit-source-id: 2612f27d08b78e5dddc99b3ce663bdee564f3bae

glean/lang/csharp/discovery/run.sh

@@ -38,6 +38,7 @@ out=$(mktemp -d)
 enable_discovery_buck=default
 enable_discovery_msbuild=default
 enable_discovery_unity=default
+index_project_files=default
 
 # The maximum number of (materialized) work-items that will be indexed
 limit=none
@@ -120,6 +121,10 @@ while [[ "$#" -gt 0 ]]; do
     --deterministic)
     deterministic=true
     ;;
+    --index-project-files)
+    index_project_files="$2"
+    shift
+    ;;
   esac
   shift
 done
@@ -160,6 +165,14 @@ if [ "$enable_discovery_unity" = "default" ]; then
   fi
 fi
 
+if [ "$index_project_files" = "default" ]; then
+  if jk check code_indexing/csharp:index_project_files --exit-with-code; then
+    index_project_files="true"
+  else
+    index_project_files="false"
+  fi
+fi
+
 if [ "$enable_discovery_msbuild" = "true" ]; then
   if [ ! -f "$paths" ]; then
     mkdir -p "$(dirname "$paths")"
@@ -219,7 +232,7 @@ if [ "$limit" != "none" ]; then
 fi
 
 title "Indexing work"
-$dotnet_host_path run --project "$indexer_project" -- "$materialized_work" --out "$facts" --log-level "$log_level" >&2
+$dotnet_host_path run --project "$indexer_project" -- "$materialized_work" --out "$facts" --log-level "$log_level" --index-project-files "$index_project_files" >&2
 
 echo "$facts"/*.json | xargs glean create "${glean_args[@]}"
 

glean/lang/csharp/indexer/Indexer.cs

@@ -36,7 +36,7 @@ namespace Glean.Indexer;
 
 public class Indexer
 {
-    public static void IndexWorkItem(FactStore factStore, MaterializedWorkItem workItem, string outputPath, LogEventLevel logLevel)
+    public static void IndexWorkItem(FactStore factStore, MaterializedWorkItem workItem, string outputPath, LogEventLevel logLevel, bool indexProjectFiles)
     {
         Log.Information($"Discovered work-item:\n{JsonSerializer.Serialize(workItem, new JsonSerializerOptions { WriteIndented = true })}");
 
@@ -45,9 +45,12 @@ public static void IndexWorkItem(FactStore factStore, MaterializedWorkItem workI
             case MaterializedWorkItem.MSBuildProject msbuildProjectWorkItem:
             {
                 var projectPath = msbuildProjectWorkItem.ProjectPath;
-                var projectFact = ProjectFact.MSBuild(msbuildProjectWorkItem);
-                factStore.Add(projectFact);
-                IndexProjectFile(factStore, projectPath, projectFact);
+                if (indexProjectFiles)
+                {
+                    var projectFact = ProjectFact.MSBuild(msbuildProjectWorkItem);
+                    factStore.Add(projectFact);
+                    IndexProjectFile(factStore, projectPath, projectFact);
+                }
                 BuildAndIndexProject(factStore, projectPath, outputPath, logLevel);
                 break;
             }
@@ -58,10 +61,13 @@ public static void IndexWorkItem(FactStore factStore, MaterializedWorkItem workI
                 foreach (var projectPath in msbuildSolutionWorkItem.ProjectPaths)
                 {
                     var msbuildProjectWorkItem = new MaterializedWorkItem.MSBuildProject(projectPath);
-                    var projectFact = ProjectFact.MSBuild(msbuildProjectWorkItem);
-                    var solutionToProjectFactKey = new SolutionToProjectFactKey(solutionFact, projectFact);
-                    factStore.Add(new SolutionToProjectFact(solutionToProjectFactKey));
-                    IndexProjectFile(factStore, projectPath, projectFact);
+                    if (indexProjectFiles)
+                    {
+                        var projectFact = ProjectFact.MSBuild(msbuildProjectWorkItem);
+                        var solutionToProjectFactKey = new SolutionToProjectFactKey(solutionFact, projectFact);
+                        factStore.Add(new SolutionToProjectFact(solutionToProjectFactKey));
+                        IndexProjectFile(factStore, projectPath, projectFact);
+                    }
                     BuildAndIndexProject(factStore, projectPath, outputPath, logLevel);
                 }
 
@@ -70,9 +76,12 @@ public static void IndexWorkItem(FactStore factStore, MaterializedWorkItem workI
             case MaterializedWorkItem.UnityPackage unityPackageWorkItem:
             {
                 var projectPath = unityPackageWorkItem.GeneratedProjectPath;
-                var projectFact = ProjectFact.Unity(unityPackageWorkItem);
-                factStore.Add(projectFact);
-                IndexProjectFile(factStore, projectPath, projectFact);
+                if (indexProjectFiles)
+                {
+                    var projectFact = ProjectFact.Unity(unityPackageWorkItem);
+                    factStore.Add(projectFact);
+                    IndexProjectFile(factStore, projectPath, projectFact);
+                }
                 BuildAndIndexProject(factStore, projectPath, outputPath, logLevel);
                 break;
             }

glean/lang/csharp/indexer/Program.cs

@@ -32,6 +32,12 @@ public static void Main(string[] args)
             description: "The path of a file containing work for the indexer, encoded as a JSON array"
         );
 
+        var indexProjectFilesOption = new Option<bool>(
+            name: "--index-project-files",
+            description: "Include build metadata in the index",
+            getDefaultValue: () => true
+        );
+
         var outOption = new Option<string>(
             aliases: new [] { "--out", "-o" },
             description: "The path of a directory into which the indexer will write facts",
@@ -66,11 +72,12 @@ public static void Main(string[] args)
             workPathArgument,
             outOption,
             logLevelOption,
+            indexProjectFilesOption
         };
 
         rootCommand.SetHandler
         (
-            (workPath, outputPath, logLevel) =>
+            (workPath, outputPath, logLevel, indexProjectFiles) =>
             {
                 Log.Logger = new LoggerConfiguration()
                     .MinimumLevel
@@ -96,6 +103,11 @@ public static void Main(string[] args)
                     return;
                 }
 
+                if (!indexProjectFiles)
+                {
+                    Log.Warning($"Indexing of project files is disabled");
+                }
+
                 Log.Information("Initializing MSBuild");
                 Build.Initialize();
 
@@ -106,14 +118,15 @@ public static void Main(string[] args)
 
                 foreach (var workItem in work)
                 {
-                    Indexer.IndexWorkItem(factStore, workItem, outputPath, logLevel);
+                    Indexer.IndexWorkItem(factStore, workItem, outputPath, logLevel, indexProjectFiles);
                 }
 
                 factStore.Flush();
             },
             workPathArgument,
             outOption,
-            logLevelOption
+            logLevelOption,
+            indexProjectFilesOption
         );
 
         rootCommand.Invoke(args);