Add support for configuring knot in pyproject.toml files (#15493)

## Summary

This PR adds support for configuring Red Knot in the `tool.knot` section
of the project's
`pyproject.toml` section. Options specified on the CLI precede the
options in the configuration file.

This PR only supports the `environment` and the `src.root` options for
now.
Other options will be added as separate PRs.

There are also a few concerns that I intentionally ignored as part of
this PR:

* Handling of relative paths: We need to anchor paths relative to the
current working directory (CLI), or the project (`pyproject.toml` or
`knot.toml`)
* Tracking the source of a value. Diagnostics would benefit from knowing
from which configuration a value comes so that we can point the user to
the right configuration file (or CLI) if the configuration is invalid.
* Schema generation and there's a lot more; see
#15491

This PR changes the default for first party codes: Our existing default
was to only add the project root. Now, Red Knot adds the project root
and `src` (if such a directory exists).

Theoretically, we'd have to add a file watcher event that changes the
first-party search paths if a user later creates a `src` directory. I
think this is pretty uncommon, which is why I ignored the complexity for
now but I can be persuaded to handle it if it's considered important.

Part of #15491

## Test Plan

Existing tests, new file watching test demonstrating that changing the
python version and platform is correctly reflected.

crates/red_knot/Cargo.toml

@@ -32,9 +32,14 @@ tracing-flame = { workspace = true }
 tracing-tree = { workspace = true }
 
 [dev-dependencies]
+ruff_db = { workspace = true, features = ["testing"] }
+
+insta = { workspace = true, features = ["filters"] }
+insta-cmd = { workspace = true }
 filetime = { workspace = true }
+regex = { workspace = true }
 tempfile = { workspace = true }
-ruff_db = { workspace = true, features = ["testing"] }
+toml = { workspace = true }
 
 [lints]
 workspace = true

crates/red_knot/src/main.rs

@@ -9,7 +9,7 @@ use python_version::PythonVersion;
 use red_knot_python_semantic::SitePackages;
 use red_knot_server::run_server;
 use red_knot_workspace::db::ProjectDatabase;
-use red_knot_workspace::project::settings::Configuration;
+use red_knot_workspace::project::options::{EnvironmentOptions, Options};
 use red_knot_workspace::project::ProjectMetadata;
 use red_knot_workspace::watch;
 use red_knot_workspace::watch::ProjectWatcher;
@@ -71,31 +71,30 @@ struct Args {
 }
 
 impl Args {
-    fn to_configuration(&self, cli_cwd: &SystemPath) -> Configuration {
-        let mut configuration = Configuration::default();
-
-        if let Some(python_version) = self.python_version {
-            configuration.python_version = Some(python_version.into());
-        }
-
-        if let Some(venv_path) = &self.venv_path {
-            configuration.search_paths.site_packages = Some(SitePackages::Derived {
-                venv_path: SystemPath::absolute(venv_path, cli_cwd),
-            });
-        }
-
-        if let Some(typeshed) = &self.typeshed {
-            configuration.search_paths.typeshed = Some(SystemPath::absolute(typeshed, cli_cwd));
+    fn to_options(&self, cli_cwd: &SystemPath) -> Options {
+        Options {
+            environment: Some(EnvironmentOptions {
+                python_version: self.python_version.map(Into::into),
+                venv_path: self
+                    .venv_path
+                    .as_ref()
+                    .map(|venv_path| SitePackages::Derived {
+                        venv_path: SystemPath::absolute(venv_path, cli_cwd),
+                    }),
+                typeshed: self
+                    .typeshed
+                    .as_ref()
+                    .map(|typeshed| SystemPath::absolute(typeshed, cli_cwd)),
+                extra_paths: self.extra_search_path.as_ref().map(|extra_search_paths| {
+                    extra_search_paths
+                        .iter()
+                        .map(|path| SystemPath::absolute(path, cli_cwd))
+                        .collect()
+                }),
+                ..EnvironmentOptions::default()
+            }),
+            ..Default::default()
         }
-
-        if let Some(extra_search_paths) = &self.extra_search_path {
-            configuration.search_paths.extra_paths = extra_search_paths
-                .iter()
-                .map(|path| Some(SystemPath::absolute(path, cli_cwd)))
-                .collect();
-        }
-
-        configuration
     }
 }
 
@@ -164,18 +163,13 @@ fn run() -> anyhow::Result<ExitStatus> {
         .unwrap_or_else(|| cli_base_path.clone());
 
     let system = OsSystem::new(cwd.clone());
-    let cli_configuration = args.to_configuration(&cwd);
-    let workspace_metadata = ProjectMetadata::discover(
-        system.current_directory(),
-        &system,
-        Some(&cli_configuration),
-    )?;
-
-    // TODO: Use the `program_settings` to compute the key for the database's persistent
-    //   cache and load the cache if it exists.
+    let cli_options = args.to_options(&cwd);
+    let mut workspace_metadata = ProjectMetadata::discover(system.current_directory(), &system)?;
+    workspace_metadata.apply_cli_options(cli_options.clone());
+
     let mut db = ProjectDatabase::new(workspace_metadata, system)?;
 
-    let (main_loop, main_loop_cancellation_token) = MainLoop::new(cli_configuration);
+    let (main_loop, main_loop_cancellation_token) = MainLoop::new(cli_options);
 
     // Listen to Ctrl+C and abort the watch mode.
     let main_loop_cancellation_token = Mutex::new(Some(main_loop_cancellation_token));
@@ -228,19 +222,19 @@ struct MainLoop {
     /// The file system watcher, if running in watch mode.
     watcher: Option<ProjectWatcher>,
 
-    cli_configuration: Configuration,
+    cli_options: Options,
 }
 
 impl MainLoop {
-    fn new(cli_configuration: Configuration) -> (Self, MainLoopCancellationToken) {
+    fn new(cli_options: Options) -> (Self, MainLoopCancellationToken) {
         let (sender, receiver) = crossbeam_channel::bounded(10);
 
         (
             Self {
                 sender: sender.clone(),
                 receiver,
                 watcher: None,
-                cli_configuration,
+                cli_options,
             },
             MainLoopCancellationToken { sender },
         )
@@ -324,7 +318,7 @@ impl MainLoop {
                 MainLoopMessage::ApplyChanges(changes) => {
                     revision += 1;
                     // Automatically cancels any pending queries and waits for them to complete.
-                    db.apply_changes(changes, Some(&self.cli_configuration));
+                    db.apply_changes(changes, Some(&self.cli_options));
                     if let Some(watcher) = self.watcher.as_mut() {
                         watcher.update(db);
                     }

crates/red_knot/tests/cli.rs

@@ -0,0 +1,60 @@
+use anyhow::Context;
+use insta_cmd::{assert_cmd_snapshot, get_cargo_bin};
+use std::process::Command;
+use tempfile::TempDir;
+
+/// Specifying an option on the CLI should take precedence over the same setting in the
+/// project's configuration.
+#[test]
+fn test_config_override() -> anyhow::Result<()> {
+    let tempdir = TempDir::new()?;
+
+    std::fs::write(
+        tempdir.path().join("pyproject.toml"),
+        r#"
+[tool.knot.environment]
+python-version = "3.11"
+"#,
+    )
+    .context("Failed to write settings")?;
+
+    std::fs::write(
+        tempdir.path().join("test.py"),
+        r#"
+import sys
+
+# Access `sys.last_exc` that was only added in Python 3.12
+print(sys.last_exc)
+"#,
+    )
+    .context("Failed to write test.py")?;
+
+    insta::with_settings!({filters => vec![(&*tempdir_filter(&tempdir), "<temp_dir>/")]}, {
+        assert_cmd_snapshot!(knot().arg("--project").arg(tempdir.path()), @r"
+        success: false
+        exit_code: 1
+        ----- stdout -----
+        error[lint:unresolved-attribute] <temp_dir>/test.py:5:7 Type `<module 'sys'>` has no attribute `last_exc`
+
+        ----- stderr -----
+        ");
+    });
+
+    assert_cmd_snapshot!(knot().arg("--project").arg(tempdir.path()).arg("--python-version").arg("3.12"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+
+    ----- stderr -----
+    ");
+
+    Ok(())
+}
+
+fn knot() -> Command {
+    Command::new(get_cargo_bin("red_knot"))
+}
+
+fn tempdir_filter(tempdir: &TempDir) -> String {
+    format!(r"{}\\?/?", regex::escape(tempdir.path().to_str().unwrap()))
+}